From 78067ced843ba0666885ce9122c3a963617b2887 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 20:58:18 -0600 Subject: [PATCH 01/33] mp: allow directly built CFL log chunks Signed-off-by: Eduardo Silva --- src/flb_mp.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/flb_mp.c b/src/flb_mp.c index d02e91281da..391aaa02cab 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -2008,7 +2008,11 @@ struct flb_mp_chunk_cobj *flb_mp_chunk_cobj_create(struct flb_log_event_encoder { struct flb_mp_chunk_cobj *chunk_cobj; - if (!log_encoder || !log_decoder) { + /* + * Both codecs can be NULL for a chunk whose records are built directly as + * CFL objects, the codecs are bound later by the processor that runs it. + */ + if ((log_encoder == NULL) != (log_decoder == NULL)) { return NULL; } @@ -2061,7 +2065,7 @@ int flb_mp_chunk_cobj_encode_append(struct flb_mp_chunk_cobj *chunk_cobj) msgpack_sbuffer buffer; msgpack_packer packer; - if (!chunk_cobj) { + if (!chunk_cobj || !chunk_cobj->log_encoder) { return -1; } @@ -2399,7 +2403,10 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, * populate the cobj list. Otherwise it means all the content is ready * as a chunk_cobj_record. */ - bytes = chunk_cobj->log_decoder->length - chunk_cobj->log_decoder->offset; + bytes = 0; + if (chunk_cobj->log_decoder) { + bytes = chunk_cobj->log_decoder->length - chunk_cobj->log_decoder->offset; + } if (bytes > 0) { record = flb_mp_chunk_record_create(chunk_cobj); From 7444e3361521efd2b79ec00cd462a775992f4ec6 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 20:58:18 -0600 Subject: [PATCH 02/33] processor: support worker-safe CFL log processing Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_processor.h | 18 +++ src/flb_processor.c | 246 +++++++++++++++++++++-------- 2 files changed, 200 insertions(+), 64 deletions(-) diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index 6d0b6eab6ea..d177052eca1 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -65,8 +65,15 @@ * FLB_PROCESSOR_FLAG_LOG_WINDOWS: the logs callback only depends on each * record and its group, so the processor chain can run it over large buffers * in group aligned windows. + * + * FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE: the logs callback is safe to invoke + * from an input protocol worker. It must not depend on the caller's event + * loop, scheduler, coroutine or thread-local state, emit records through + * thread-bound APIs, or require ordering between independently received + * batches. */ #define FLB_PROCESSOR_FLAG_LOG_WINDOWS 1 +#define FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE (1 << 1) /* The current values mean the processor stack will @@ -89,6 +96,7 @@ struct flb_log_event_decoder; struct flb_log_event_encoder; struct flb_processor_instance; struct flb_condition; +struct flb_mp_chunk_cobj; struct flb_processor_unit { int event_type; @@ -276,6 +284,16 @@ int flb_processor_run_with_count(struct flb_processor *proc, size_t starting_sta void **out_buf, size_t *out_size, int input_records, int *output_records); +/* Run the logs processors over a chunk of CFL records, consuming the chunk */ +int flb_processor_logs_chunk_supported(struct flb_processor *proc, + int worker_thread); +int flb_processor_run_logs_chunk(struct flb_processor *proc, + const char *tag, size_t tag_len, + struct flb_mp_chunk_cobj *chunk, + int worker_thread, + void **out_buf, size_t *out_size, + int *output_records); + struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, int event_type, char *unit_name); diff --git a/src/flb_processor.c b/src/flb_processor.c index 8e68a2353a1..59a4a89ea04 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -1465,6 +1465,37 @@ static void chunk_codec_destroy(struct chunk_codec *codec) codec->ready = FLB_FALSE; } +#define CHUNK_MATERIALIZE_OK 0 +#define CHUNK_MATERIALIZE_PARTIAL 1 + +/* + * Decode every record while the per-run codec is private to this call. Keep + * the decoded prefix on an error, matching the former lazy callback behavior. + * record_next() does not distinguish malformed input from allocation/CFL + * conversion failures, so all three use prefix semantics here. + */ +static int chunk_cobj_materialize(struct flb_mp_chunk_cobj *chunk_cobj) +{ + int ret; + int records; + struct flb_mp_chunk_record *record; + + while (1) { + ret = flb_mp_chunk_cobj_record_next(chunk_cobj, &record); + if (ret == FLB_MP_CHUNK_RECORD_EOF) { + return CHUNK_MATERIALIZE_OK; + } + else if (ret != FLB_MP_CHUNK_RECORD_OK) { + records = flb_mp_chunk_cobj_count_log_records(chunk_cobj); + flb_warn("[processor] failed to decode record, processing %d decoded records", + records); + flb_log_event_decoder_reset(chunk_cobj->log_decoder, NULL, 0); + chunk_cobj->record_pos = NULL; + return CHUNK_MATERIALIZE_PARTIAL; + } + } +} + /* * Large log buffers, such as complete chunks reaching output processors, are * processed in windows when every native unit of the run is record local: @@ -1535,9 +1566,9 @@ static struct log_window *log_windows_scan(void *data, size_t data_size, int *co FLB_PROCESSOR_LOG_WINDOW_BYTES, FLB_PROCESSOR_LOG_WINDOW_RECORDS, &end, &records) != 0) { - /* the single chunk path reports invalid content as usual */ - flb_free(windows); - return NULL; + /* Let the final window preserve records before a corrupt tail. */ + end = data_size; + records = 0; } if (*count == capacity) { @@ -1657,7 +1688,12 @@ static int run_log_windows(struct flb_processor *proc, struct chunk_codec *codec goto error; } - records = windows[w].records; + ret = chunk_cobj_materialize(chunk_cobj); + if (ret == CHUNK_MATERIALIZE_PARTIAL) { + units[0].error = FLB_TRUE; + } + + records = flb_mp_chunk_cobj_count_log_records(chunk_cobj); for (i = 0; i < unit_count; i++) { p_ins = units[i].pu->ctx; @@ -1777,6 +1813,7 @@ int flb_processor_run(struct flb_processor *proc, * or a CMetrics, CTraces or CProfiles context for the other signal types. */ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, + struct flb_mp_chunk_cobj *input_chunk, size_t starting_stage, int type, const char *tag, size_t tag_len, @@ -1791,6 +1828,7 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, int item_metrics_available; int finalize; int raw_result; + int unit_lock_held; int log_buffer_normalized = FLB_FALSE; int log_buffer_records = input_records; void *cur_buf = NULL; @@ -1851,6 +1889,18 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, cur_buf = data; cur_size = data_size; + /* a chunk built by the caller replaces the msgpack decoding step */ + if (input_chunk) { + if (chunk_codec_prepare(codec) != 0) { + flb_mp_chunk_cobj_destroy(input_chunk); + return -1; + } + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); + input_chunk->log_encoder = &codec->encoder; + input_chunk->log_decoder = &codec->decoder; + chunk_cobj = input_chunk; + } + /* iterate list units */ mk_list_foreach(head, list) { pu = mk_list_entry(head, struct flb_processor_unit, _head); @@ -1866,6 +1916,7 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, tmp_buf = NULL; tmp_size = 0; pu_error = FLB_FALSE; + unit_lock_held = FLB_FALSE; unit_in_items = -1; unit_out_items = -1; item_metrics_available = FLB_FALSE; @@ -1953,13 +2004,52 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, } } + /* + * The decoder and CFL chunk belong to this processor_run() call, not + * to a processor instance. Materialize the complete input before + * taking the unit lock so concurrent calls do not serialize codec + * work behind an otherwise record-local callback. + */ + if (type == FLB_PROCESSOR_LOGS && chunk_cobj == NULL && + pu->unit_type == FLB_PROCESSOR_UNIT_NATIVE) { + p_ins = pu->ctx; + + if (p_ins->p->cb_process_logs != NULL) { + if (chunk_codec_prepare(codec) != 0) { + if (cur_buf != data) { + flb_free(cur_buf); + } + return -1; + } + + flb_log_event_decoder_reset(&codec->decoder, cur_buf, cur_size); + chunk_cobj = flb_mp_chunk_cobj_create(&codec->encoder, &codec->decoder); + if (chunk_cobj == NULL) { + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); + if (cur_buf != data) { + flb_free(cur_buf); + } + return -1; + } + + ret = chunk_cobj_materialize(chunk_cobj); + if (ret == CHUNK_MATERIALIZE_PARTIAL) { + pu_error = FLB_TRUE; + } + } + } + ret = acquire_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); if (ret != FLB_TRUE) { + if (chunk_cobj) { + flb_mp_chunk_cobj_destroy(chunk_cobj); + } return -1; } + unit_lock_held = FLB_TRUE; /* run the unit */ if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { @@ -2102,55 +2192,6 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, if (type == FLB_PROCESSOR_LOGS) { if (p_ins->p->cb_process_logs != NULL) { - /* if no previous chunkj_cobj exist, create instance. Note that this context will last - * until no more processors exists or the next one is a "filter" type processor. - */ - if (!chunk_cobj) { - if (chunk_codec_prepare(codec) != 0) { - if (cur_buf != data) { - flb_free(cur_buf); - } - pu_error = FLB_TRUE; - processor_metrics_update(proc, pu, - metrics_scope, - metrics_owner, - metrics_signal, - unit_in_items, - unit_out_items, - item_metrics_available, - pu_error); - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - return -1; - } - flb_log_event_decoder_reset(&codec->decoder, cur_buf, cur_size); - - /* create the context */ - chunk_cobj = flb_mp_chunk_cobj_create(&codec->encoder, &codec->decoder); - if (chunk_cobj == NULL) { - flb_log_event_decoder_reset(&codec->decoder, NULL, 0); - if (cur_buf != data) { - flb_free(cur_buf); - } - - pu_error = FLB_TRUE; - processor_metrics_update(proc, pu, - metrics_scope, - metrics_owner, - metrics_signal, - unit_in_items, - unit_out_items, - item_metrics_available, - pu_error); - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - - return -1; - } - } - /* Set the processor condition if it exists */ if (pu->condition) { flb_debug("[processor] setting condition for chunk processing (pu=%s)", pu->name); @@ -2170,6 +2211,10 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, /* Clear the condition after processing */ chunk_cobj->condition = NULL; + release_lock(&pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + unit_lock_held = FLB_FALSE; finalize = FLB_FALSE; /* is this processing_unit the last one from the list ? */ @@ -2219,9 +2264,6 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, unit_out_items, item_metrics_available, pu_error); - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); return 0; } @@ -2236,10 +2278,6 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, flb_free(cur_buf); } - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - pu_error = FLB_TRUE; processor_metrics_update(proc, pu, metrics_scope, @@ -2410,9 +2448,11 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, unit_out_items, item_metrics_available, pu_error); - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); + if (unit_lock_held == FLB_TRUE) { + release_lock(&pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + } } /* set output buffer */ @@ -2485,7 +2525,7 @@ int flb_processor_run_with_count(struct flb_processor *proc, struct chunk_codec codec; codec.ready = FLB_FALSE; - ret = processor_run(proc, &codec, starting_stage, type, tag, tag_len, + ret = processor_run(proc, &codec, NULL, starting_stage, type, tag, tag_len, data, data_size, out_buf, out_size, input_records, output_records); chunk_codec_destroy(&codec); @@ -2493,6 +2533,84 @@ int flb_processor_run_with_count(struct flb_processor *proc, return ret; } +/* + * A logs chunk built directly as CFL records can enter the pipeline when the + * first unit is a native processor and every native unit processes logs. A + * worker thread additionally requires every unit to be a worker-safe native + * processor. The whole chain is checked before any unit runs so fallback can + * never execute an already-processed prefix a second time. + */ +int flb_processor_logs_chunk_supported(struct flb_processor *proc, + int worker_thread) +{ + struct mk_list *head; + struct flb_processor_unit *pu; + struct flb_processor_instance *p_ins; + + if (!proc || !proc->is_active || mk_list_is_empty(&proc->logs) == 0) { + return FLB_FALSE; + } + + mk_list_foreach(head, &proc->logs) { + pu = mk_list_entry(head, struct flb_processor_unit, _head); + if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { + if (head == proc->logs.next || worker_thread == FLB_TRUE) { + return FLB_FALSE; + } + continue; + } + + p_ins = pu->ctx; + if (p_ins->p->cb_process_logs == NULL) { + return FLB_FALSE; + } + if (worker_thread == FLB_TRUE && + (p_ins->p->flags & FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE) == 0) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + +/* + * Run the logs processors over a chunk of CFL records. The chunk is always + * consumed. On success the output buffer is owned by the caller, it is NULL + * with a zero size when every record was dropped. + */ +int flb_processor_run_logs_chunk(struct flb_processor *proc, + const char *tag, size_t tag_len, + struct flb_mp_chunk_cobj *chunk, + int worker_thread, + void **out_buf, size_t *out_size, + int *output_records) +{ + int ret; + struct chunk_codec codec; + + *out_buf = NULL; + *out_size = 0; + if (output_records) { + *output_records = -1; + } + + if (!chunk) { + return -1; + } + + if (flb_processor_logs_chunk_supported(proc, worker_thread) == FLB_FALSE) { + flb_mp_chunk_cobj_destroy(chunk); + return -1; + } + + codec.ready = FLB_FALSE; + ret = processor_run(proc, &codec, chunk, 0, FLB_PROCESSOR_LOGS, tag, tag_len, + NULL, 0, out_buf, out_size, -1, output_records); + chunk_codec_destroy(&codec); + + return ret; +} + void flb_processor_destroy(struct flb_processor *proc) { struct mk_list *head; From bc4c2cf24eb76e0174d3578a8e6b052a2fe85efe Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 20:58:18 -0600 Subject: [PATCH 03/33] input_log: ingest processed CFL log chunks Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_input_log.h | 14 +++ src/flb_input_log.c | 135 ++++++++++++++++++++++++----- 2 files changed, 125 insertions(+), 24 deletions(-) diff --git a/include/fluent-bit/flb_input_log.h b/include/fluent-bit/flb_input_log.h index fa88551bc7b..8adfb999aec 100644 --- a/include/fluent-bit/flb_input_log.h +++ b/include/fluent-bit/flb_input_log.h @@ -45,4 +45,18 @@ int flb_input_log_append_processed(struct flb_input_instance *ins, size_t tag_len, const void *buf, size_t buf_size); +int flb_input_log_append_processed_routed(struct flb_input_instance *ins, + size_t records, + const char *tag, + size_t tag_len, + const void *buf, + size_t buf_size); + +struct flb_mp_chunk_cobj; +int flb_input_log_chunk_supported(struct flb_input_instance *ins, + int worker_thread); +int flb_input_log_append_chunk(struct flb_input_instance *ins, + const char *tag, size_t tag_len, + struct flb_mp_chunk_cobj *chunk, + int worker_thread); #endif diff --git a/src/flb_input_log.c b/src/flb_input_log.c index 764c9893fc1..255fea4f369 100644 --- a/src/flb_input_log.c +++ b/src/flb_input_log.c @@ -1526,6 +1526,44 @@ static int input_log_append_processed_internal(struct flb_input_instance *ins, return ret; } +/* Store a log buffer that already went through the input processors */ +static int input_log_append_output(struct flb_input_instance *ins, size_t records, + const char *tag, size_t tag_len, + const void *buf, size_t buf_size) +{ + if (flb_input_is_threaded(ins) == FLB_TRUE && + input_has_conditional_routes(ins) == FLB_TRUE) { + return flb_input_chunk_ring_buffer_enqueue_log_routing(ins, + FLB_INPUT_LOGS, + records, + tag, tag_len, + buf, buf_size); + } + + return input_log_append_processed_internal(ins, records, + tag, tag_len, + buf, buf_size, + FLB_FALSE); +} + +/* Tag used by processors when the input does not provide one */ +static void input_log_processor_tag(struct flb_input_instance *ins, + const char **tag, size_t *tag_len) +{ + if (*tag) { + return; + } + + if (ins->tag && ins->tag_len > 0) { + *tag = ins->tag; + *tag_len = ins->tag_len; + } + else { + *tag = ins->name; + *tag_len = strlen(ins->name); + } +} + /* * 'exact_records' tells whether 'records' is the exact number of log records in * the buffer. Caller-provided hints (for example, lines read by an input) are @@ -1546,16 +1584,7 @@ static int input_log_append(struct flb_input_instance *ins, processor_is_active = flb_processor_is_active(ins->processor); if (processor_is_active) { - if (!tag) { - if (ins->tag && ins->tag_len > 0) { - tag = ins->tag; - tag_len = ins->tag_len; - } - else { - tag = ins->name; - tag_len = strlen(ins->name); - } - } + input_log_processor_tag(ins, &tag, &tag_len); if (exact_records && records <= INT_MAX) { known_records = (int) records; @@ -1589,20 +1618,7 @@ static int input_log_append(struct flb_input_instance *ins, } } - if (flb_input_is_threaded(ins) == FLB_TRUE && - input_has_conditional_routes(ins) == FLB_TRUE) { - ret = flb_input_chunk_ring_buffer_enqueue_log_routing(ins, - FLB_INPUT_LOGS, - records, - tag, tag_len, - out_buf, out_size); - } - else { - ret = input_log_append_processed_internal(ins, records, - tag, tag_len, - out_buf, out_size, - FLB_FALSE); - } + ret = input_log_append_output(ins, records, tag, tag_len, out_buf, out_size); if (processor_is_active && buf != out_buf) { flb_free(out_buf); @@ -1641,6 +1657,25 @@ int flb_input_log_append_processed(struct flb_input_instance *ins, FLB_TRUE); } +/* + * Store an already processed buffer from a thread that might not own input + * chunks. Unlike flb_input_log_append_processed(), this entry point performs + * the normal threaded-input handoff instead of forcing a local append. + */ +int flb_input_log_append_processed_routed(struct flb_input_instance *ins, + size_t records, + const char *tag, + size_t tag_len, + const void *buf, + size_t buf_size) +{ + if (records == 0) { + records = flb_mp_count_log_records(buf, buf_size); + } + + return input_log_append_output(ins, records, tag, tag_len, buf, buf_size); +} + /* Take a msgpack serialized record and enqueue it as a chunk */ int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins, size_t processor_starting_stage, @@ -1670,3 +1705,55 @@ int flb_input_log_append_records(struct flb_input_instance *ins, ret = input_log_append(ins, 0, records, FLB_FALSE, tag, tag_len, buf, buf_size); return ret; } + +/* TRUE when the input processors can take log records built as a CFL chunk */ +int flb_input_log_chunk_supported(struct flb_input_instance *ins, + int worker_thread) +{ + return flb_processor_logs_chunk_supported(ins->processor, worker_thread); +} + +/* + * Run the input processors over a chunk of CFL records and enqueue the + * result, skipping the msgpack round trip the processors would otherwise do. + * The chunk is always consumed, callers check flb_input_log_chunk_supported() + * first. + */ +int flb_input_log_append_chunk(struct flb_input_instance *ins, + const char *tag, size_t tag_len, + struct flb_mp_chunk_cobj *chunk, + int worker_thread) +{ + int ret; + int records = -1; + void *out_buf = NULL; + size_t out_size = 0; + + input_log_processor_tag(ins, &tag, &tag_len); + + ret = flb_processor_run_logs_chunk(ins->processor, tag, tag_len, chunk, + worker_thread, + &out_buf, &out_size, &records); + if (ret == -1) { + return -1; + } + + if (out_size == 0) { + flb_free(out_buf); + return 0; + } + + if (records < 0) { + records = flb_mp_count_log_records(out_buf, out_size); + } + + if (ins->ingress_queue_enabled == FLB_TRUE) { + return flb_input_ingress_queue_log_processed_take_records( + ins, records, tag, tag_len, out_buf, out_size, out_size); + } + + ret = input_log_append_output(ins, records, tag, tag_len, out_buf, out_size); + flb_free(out_buf); + + return ret; +} From ae86e333c963e81dc22316143d753b2bb3ee140f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 20:58:18 -0600 Subject: [PATCH 04/33] input_ingest: route preprocessed log events Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_input.h | 6 ++++ src/flb_input_ingest.c | 55 +++++++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/include/fluent-bit/flb_input.h b/include/fluent-bit/flb_input.h index 4824ea824af..b0086eff626 100644 --- a/include/fluent-bit/flb_input.h +++ b/include/fluent-bit/flb_input.h @@ -957,6 +957,12 @@ int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, const char *tag, size_t tag_len, void *buf, size_t buf_size, size_t allocation_size); +int flb_input_ingress_queue_log_processed_take_records( + struct flb_input_instance *ins, + size_t records, + const char *tag, size_t tag_len, + void *buf, size_t buf_size, + size_t allocation_size); int flb_input_ingress_queue_metrics(struct flb_input_instance *ins, const char *tag, size_t tag_len, struct cmt *cmt, size_t payload_size); diff --git a/src/flb_input_ingest.c b/src/flb_input_ingest.c index 91ff0e0675b..0a5bd2a2cc3 100644 --- a/src/flb_input_ingest.c +++ b/src/flb_input_ingest.c @@ -52,6 +52,7 @@ struct flb_input_ingress_event { void *buf; size_t size; size_t records; + int processed; } log; struct ctrace *traces; struct cprof *profiles; @@ -378,7 +379,16 @@ static int flb_input_ingress_collector(struct flb_input_instance *ins, mk_list_entry_init(&event->_head); if (event->type == FLB_INPUT_INGRESS_LOG) { - if (event->data.log.records > 0) { + if (event->data.log.processed == FLB_TRUE) { + result = flb_input_log_append_processed_routed( + ins, + event->data.log.records, + event->tag, + event->tag != NULL ? flb_sds_len(event->tag) : 0, + event->data.log.buf, + event->data.log.size); + } + else if (event->data.log.records > 0) { result = flb_input_log_append_records( ins, event->data.log.records, @@ -571,13 +581,14 @@ int flb_input_ingress_queue_log_take(struct flb_input_instance *ins, allocation_size); } -int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, - size_t records, - const char *tag, - size_t tag_len, - void *buf, - size_t buf_size, - size_t allocation_size) +static int flb_input_ingress_queue_log_take_internal(struct flb_input_instance *ins, + size_t records, + int processed, + const char *tag, + size_t tag_len, + void *buf, + size_t buf_size, + size_t allocation_size) { struct flb_input_ingress_event *event; @@ -597,6 +608,7 @@ int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, event->data.log.buf = buf; event->data.log.size = buf_size; event->data.log.records = records; + event->data.log.processed = processed; if (allocation_size < buf_size) { allocation_size = buf_size; } @@ -605,6 +617,33 @@ int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, return flb_input_ingress_enqueue(ins, event); } +int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, + size_t records, + const char *tag, + size_t tag_len, + void *buf, + size_t buf_size, + size_t allocation_size) +{ + return flb_input_ingress_queue_log_take_internal(ins, records, FLB_FALSE, + tag, tag_len, buf, buf_size, + allocation_size); +} + +int flb_input_ingress_queue_log_processed_take_records( + struct flb_input_instance *ins, + size_t records, + const char *tag, + size_t tag_len, + void *buf, + size_t buf_size, + size_t allocation_size) +{ + return flb_input_ingress_queue_log_take_internal(ins, records, FLB_TRUE, + tag, tag_len, buf, buf_size, + allocation_size); +} + int flb_input_ingress_queue_metrics(struct flb_input_instance *ins, const char *tag, size_t tag_len, From dc401f5ac52e0db60acca662b141dc4820c1d8bf Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 20:58:18 -0600 Subject: [PATCH 05/33] processor_content_modifier: mark log callbacks worker safe Signed-off-by: Eduardo Silva --- plugins/processor_content_modifier/cm.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/processor_content_modifier/cm.c b/plugins/processor_content_modifier/cm.c index 5482256b9ef..db743ad05e6 100644 --- a/plugins/processor_content_modifier/cm.c +++ b/plugins/processor_content_modifier/cm.c @@ -169,5 +169,6 @@ struct flb_processor_plugin processor_content_modifier_plugin = { .cb_process_traces = cb_process_traces, .cb_exit = cb_exit, .config_map = config_map, - .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS | + FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE }; From efc96226b335848272fbcfc85a6d9cecf8b537e9 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 20:58:19 -0600 Subject: [PATCH 06/33] processor_sql: mark log callbacks worker safe Signed-off-by: Eduardo Silva --- plugins/processor_sql/sql.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/processor_sql/sql.c b/plugins/processor_sql/sql.c index aa3b1c99d22..e7f4619656f 100644 --- a/plugins/processor_sql/sql.c +++ b/plugins/processor_sql/sql.c @@ -709,6 +709,6 @@ struct flb_processor_plugin processor_sql_plugin = { .cb_process_traces = NULL, .cb_exit = cb_exit, .config_map = config_map, - .flags = 0 + .flags = FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE }; From a61d0b6a6baf96a87c6bb5b6fb23dd5661997e9a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 21:32:35 -0600 Subject: [PATCH 07/33] opentelemetry: share protobuf log decoding Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_opentelemetry.h | 42 + src/CMakeLists.txt | 2 + .../flb_opentelemetry_logs_proto.c | 1718 +++++++++++++++++ .../flb_opentelemetry_protobuf.c | 134 ++ 4 files changed, 1896 insertions(+) create mode 100644 src/opentelemetry/flb_opentelemetry_logs_proto.c create mode 100644 src/opentelemetry/flb_opentelemetry_protobuf.c diff --git a/include/fluent-bit/flb_opentelemetry.h b/include/fluent-bit/flb_opentelemetry.h index cdb529e0aef..f36e55d1415 100644 --- a/include/fluent-bit/flb_opentelemetry.h +++ b/include/fluent-bit/flb_opentelemetry.h @@ -25,6 +25,7 @@ #include #include #include +#include /* Error code values from flb_opentelemetry logs/traces helpers */ #define FLB_OTEL_LOGS_ERR_GENERIC_ERROR -1 @@ -119,6 +120,16 @@ struct flb_opentelemetry_otlp_logs_options { int logs_body_key_attributes; }; +/* Options shared by OTLP/HTTP protobuf log decoders. */ +struct flb_opentelemetry_logs_decode_options { + const char *metadata_key; + const char *body_key; + size_t output_limit; +}; + +struct flb_mp_chunk_cobj; +struct flb_opentelemetry_logs_protobuf_cfl; + /* Backward-compatible alias for older external callers. */ #define flb_opentelemetry_otlp_json_options flb_opentelemetry_otlp_logs_options @@ -204,6 +215,37 @@ int flb_opentelemetry_logs_json_to_msgpack(struct flb_log_event_encoder *encoder const char *logs_body_key, int *error_status); +int flb_opentelemetry_protobuf_validate( + const ProtobufCMessageDescriptor *descriptor, + const void *data, size_t size); + +int flb_opentelemetry_logs_protobuf_to_msgpack( + struct flb_log_event_encoder *encoder, + const void *payload, size_t payload_size, + const struct flb_opentelemetry_logs_decode_options *options, + size_t *record_count); + +int flb_opentelemetry_logs_protobuf_to_cfl( + const void *payload, size_t payload_size, + const struct flb_opentelemetry_logs_decode_options *options, + struct flb_opentelemetry_logs_protobuf_cfl **result); + +/* + * The returned result owns both the decoded protobuf arena and the CFL chunk. + * Values in the CFL chunk can borrow strings and byte arrays from that arena. + * After taking the chunk, the caller must consume or destroy it synchronously + * before destroying the result. Neither object can cross an asynchronous + * queue boundary; only the final owned MessagePack buffer may do so. + */ +struct flb_mp_chunk_cobj *flb_opentelemetry_logs_protobuf_cfl_take_chunk( + struct flb_opentelemetry_logs_protobuf_cfl *result); + +size_t flb_opentelemetry_logs_protobuf_cfl_get_encoded_size( + struct flb_opentelemetry_logs_protobuf_cfl *result); + +void flb_opentelemetry_logs_protobuf_cfl_destroy( + struct flb_opentelemetry_logs_protobuf_cfl *result); + int flb_opentelemetry_metrics_json_to_cmt(struct cfl_list *context_list, const char *body, size_t len); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1d0fa9dbde1..819a6848204 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -128,7 +128,9 @@ endif() set(src ${src} opentelemetry/flb_opentelemetry_logs.c + opentelemetry/flb_opentelemetry_logs_proto.c opentelemetry/flb_opentelemetry_metrics.c + opentelemetry/flb_opentelemetry_protobuf.c opentelemetry/flb_opentelemetry_otlp_json.c opentelemetry/flb_opentelemetry_otlp_proto.c opentelemetry/flb_opentelemetry_traces.c diff --git a/src/opentelemetry/flb_opentelemetry_logs_proto.c b/src/opentelemetry/flb_opentelemetry_logs_proto.c new file mode 100644 index 00000000000..3cc9bd99e0f --- /dev/null +++ b/src/opentelemetry/flb_opentelemetry_logs_proto.c @@ -0,0 +1,1718 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +/* + * Small requests use protobuf-c's default allocator to avoid reserving an + * arena chunk. Realistic batched requests use the arena to amortize the large + * number of short-lived allocations performed by protobuf-c. + */ +#define OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE 8192 +#define OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE 4096 +#define OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE 65536 + +static void *protobuf_arena_chunk_malloc(void *context, size_t size) +{ + (void) context; + + return flb_malloc(size); +} +static void protobuf_arena_chunk_free(void *context, void *pointer) +{ + (void) context; + + flb_free(pointer); +} + +static void *protobuf_arena_alloc(void *allocator_data, size_t size) +{ + struct cfl_arena *arena; + + arena = allocator_data; + if (size == 0) { + size = 1; + } + + return cfl_arena_malloc(arena, size); +} + +static void protobuf_arena_free(void *allocator_data, void *pointer) +{ + (void) allocator_data; + (void) pointer; +} + +static struct cfl_arena *protobuf_arena_create(void) +{ + struct cfl_arena_options options; + + cfl_arena_options_init(&options); + options.chunk_size = OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE; + options.maximum_chunk_size = OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE; + options.malloc_fn = protobuf_arena_chunk_malloc; + options.free_fn = protobuf_arena_chunk_free; + + return cfl_arena_create_with_options(&options); +} + +static Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest * +protobuf_logs_unpack(ProtobufCAllocator *allocator, size_t size, const uint8_t *data) +{ + if (flb_opentelemetry_protobuf_validate( + &opentelemetry__proto__collector__logs__v1__export_logs_service_request__descriptor, + data, size) != 0) { + return NULL; + } + return opentelemetry__proto__collector__logs__v1__export_logs_service_request__unpack( + allocator, size, data); +} + +static void protobuf_logs_free( + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *logs, + ProtobufCAllocator *allocator) +{ + opentelemetry__proto__collector__logs__v1__export_logs_service_request__free_unpacked( + logs, allocator); +} + +/* Unpack a logs request, using an arena for realistic batched payloads */ +static Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest * +protobuf_logs_decode(uint8_t *in_buf, size_t in_size, + struct cfl_arena **protobuf_arena, + ProtobufCAllocator *arena_allocator, + ProtobufCAllocator **protobuf_allocator) +{ + *protobuf_arena = NULL; + *protobuf_allocator = NULL; + + if (in_size >= OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE) { + *protobuf_arena = protobuf_arena_create(); + if (*protobuf_arena != NULL) { + arena_allocator->alloc = protobuf_arena_alloc; + arena_allocator->free = protobuf_arena_free; + arena_allocator->allocator_data = *protobuf_arena; + *protobuf_allocator = arena_allocator; + } + } + + return protobuf_logs_unpack(*protobuf_allocator, in_size, in_buf); +} + +/* + * OTLP encoding functions to pack the log records as msgpack + * ---------------------------------------------------------- + */ +static int otlp_pack_any_value(msgpack_packer *mp_pck, Opentelemetry__Proto__Common__V1__AnyValue *body); + +static int otel_pack_string(msgpack_packer *mp_pck, char *str) +{ + return msgpack_pack_str_with_body(mp_pck, str, strlen(str)); +} + +static int otel_pack_bool(msgpack_packer *mp_pck, bool val) +{ + if (val) { + return msgpack_pack_true(mp_pck); + } + else { + return msgpack_pack_false(mp_pck); + } +} + +static int otel_pack_int(msgpack_packer *mp_pck, int64_t val) +{ + return msgpack_pack_int64(mp_pck, val); +} + +static int otel_pack_double(msgpack_packer *mp_pck, double val) +{ + return msgpack_pack_double(mp_pck, val); +} + +static int otel_pack_kvarray(msgpack_packer *mp_pck, + Opentelemetry__Proto__Common__V1__KeyValue **kv_array, + size_t kv_count) +{ + int result; + int index; + + result = msgpack_pack_map(mp_pck, kv_count); + + if (result != 0) { + return result; + } + + for (index = 0; index < kv_count && result == 0; index++) { + result = otel_pack_string(mp_pck, kv_array[index]->key); + + if(result == 0) { + result = otlp_pack_any_value(mp_pck, kv_array[index]->value); + } + } + + return result; +} + +static int otel_pack_kvlist(msgpack_packer *mp_pck, + Opentelemetry__Proto__Common__V1__KeyValueList *kv_list) +{ + int kv_index; + int ret; + char *key; + Opentelemetry__Proto__Common__V1__AnyValue *value; + + ret = msgpack_pack_map(mp_pck, kv_list->n_values); + if (ret != 0) { + return ret; + } + + for (kv_index = 0; kv_index < kv_list->n_values && ret == 0; kv_index++) { + key = kv_list->values[kv_index]->key; + value = kv_list->values[kv_index]->value; + + ret = otel_pack_string(mp_pck, key); + + if(ret == 0) { + ret = otlp_pack_any_value(mp_pck, value); + } + } + + return ret; +} + +static int otel_pack_array(msgpack_packer *mp_pck, + Opentelemetry__Proto__Common__V1__ArrayValue *array) +{ + int ret; + int array_index; + + ret = msgpack_pack_array(mp_pck, array->n_values); + + if (ret != 0) { + return ret; + } + + for (array_index = 0; array_index < array->n_values && ret == 0; array_index++) { + ret = otlp_pack_any_value(mp_pck, array->values[array_index]); + } + + return ret; +} + +static int otel_pack_bytes(msgpack_packer *mp_pck, + ProtobufCBinaryData bytes) +{ + return msgpack_pack_bin_with_body(mp_pck, bytes.data, bytes.len); +} + +static int otlp_pack_any_value(msgpack_packer *mp_pck, + Opentelemetry__Proto__Common__V1__AnyValue *body) +{ + int result; + + result = -2; + + if (body == NULL) { + msgpack_pack_nil(mp_pck); + return 0; + } + + switch(body->value_case){ + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: + result = otel_pack_string(mp_pck, body->string_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE_STRINDEX: + /* Profiling-only string dictionary reference: ignore in logs. */ + result = msgpack_pack_nil(mp_pck); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: + result = otel_pack_bool(mp_pck, body->bool_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: + result = otel_pack_int(mp_pck, body->int_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: + result = otel_pack_double(mp_pck, body->double_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: + result = otel_pack_array(mp_pck, body->array_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: + result = otel_pack_kvlist(mp_pck, body->kvlist_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: + result = otel_pack_bytes(mp_pck, body->bytes_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET: + /* treat an unset value as null */ + result = msgpack_pack_nil(mp_pck); + break; + + default: + break; + } + + if (result == -2) { + flb_error("[otel]: invalid value type in pack_any_value"); + result = -1; + } + + return result; +} + +/* https://opentelemetry.io/docs/specs/otel/logs/data-model/#log-and-event-record-definition */ +static int otel_pack_v1_metadata( + const struct flb_opentelemetry_logs_decode_options *options, + msgpack_packer *mp_pck, + struct Opentelemetry__Proto__Logs__V1__LogRecord *log_record, + Opentelemetry__Proto__Resource__V1__Resource *resource, + Opentelemetry__Proto__Common__V1__InstrumentationScope *scope) +{ + int ret; + int len; + struct flb_mp_map_header mh; + struct flb_mp_map_header otlp_mh; + + flb_mp_map_header_init(&otlp_mh, mp_pck); + + len = strlen(options->metadata_key); + + /* otlp key start */ + flb_mp_map_header_append(&otlp_mh); + + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, options->metadata_key, len); + + flb_mp_map_header_init(&mh, mp_pck); + + if (log_record->observed_time_unix_nano != 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 18); + msgpack_pack_str_body(mp_pck, "observed_timestamp", 18); + msgpack_pack_uint64(mp_pck, log_record->observed_time_unix_nano); + } + + /* Value of 0 indicates unknown or missing timestamp. */ + if (log_record->time_unix_nano != 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 9); + msgpack_pack_str_body(mp_pck, "timestamp", 9); + msgpack_pack_uint64(mp_pck, log_record->time_unix_nano); + } + + /* https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber */ + if (log_record->severity_number >= 1 && log_record->severity_number <= 24) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 15); + msgpack_pack_str_body(mp_pck, "severity_number", 15); + msgpack_pack_uint64(mp_pck, log_record->severity_number); + } + + if (log_record->severity_text != NULL && strlen(log_record->severity_text) > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 13); + msgpack_pack_str_body(mp_pck, "severity_text", 13); + msgpack_pack_str(mp_pck, strlen(log_record->severity_text)); + msgpack_pack_str_body(mp_pck, log_record->severity_text, strlen(log_record->severity_text)); + } + + if (log_record->n_attributes > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "attributes", 10); + ret = otel_pack_kvarray(mp_pck, + log_record->attributes, + log_record->n_attributes); + if (ret != 0) { + return ret; + } + } + + if (log_record->dropped_attributes_count > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 24); + msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); + msgpack_pack_uint64(mp_pck, log_record->dropped_attributes_count); + } + + if (log_record->trace_id.len > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 8); + msgpack_pack_str_body(mp_pck, "trace_id", 8); + ret = otel_pack_bytes(mp_pck, log_record->trace_id); + if (ret != 0) { + return ret; + } + } + + if (log_record->span_id.len > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 7); + msgpack_pack_str_body(mp_pck, "span_id", 7); + ret = otel_pack_bytes(mp_pck, log_record->span_id); + if (ret != 0) { + return ret; + } + } + + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 11); + msgpack_pack_str_body(mp_pck, "trace_flags", 11); + msgpack_pack_uint8(mp_pck, (uint8_t) log_record->flags & 0xff); + + if (log_record->event_name != NULL && strlen(log_record->event_name) > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "event_name", 10); + msgpack_pack_str(mp_pck, strlen(log_record->event_name)); + msgpack_pack_str_body(mp_pck, log_record->event_name, strlen(log_record->event_name)); + } + + flb_mp_map_header_end(&mh); + + /* otlp key end */ + flb_mp_map_header_end(&otlp_mh); + + return 0; +} + +int flb_opentelemetry_logs_protobuf_to_msgpack( + struct flb_log_event_encoder *encoder, + const void *payload, size_t payload_size, + const struct flb_opentelemetry_logs_decode_options *options, + size_t *record_count) +{ + int ret = 0; + int len; + int resource_logs_index; + int scope_log_index; + int log_record_index; + char *logs_body_key; + int scope_has_schema_url; + struct cfl_arena *protobuf_arena; + struct flb_mp_map_header mh; + struct flb_mp_map_header mh_tmp; + struct flb_time tm; + ProtobufCAllocator arena_allocator; + ProtobufCAllocator *protobuf_allocator; + + msgpack_packer *mp_pck; + msgpack_packer *mp_pck_meta; + + /* OTel proto suff */ + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *input_logs; + Opentelemetry__Proto__Logs__V1__ScopeLogs **scope_logs; + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; + Opentelemetry__Proto__Common__V1__InstrumentationScope *scope; + + Opentelemetry__Proto__Logs__V1__ResourceLogs **resource_logs; + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; + Opentelemetry__Proto__Logs__V1__LogRecord **log_records; + Opentelemetry__Proto__Resource__V1__Resource *resource; + + mp_pck = &encoder->body.packer; + mp_pck_meta = &encoder->metadata.packer; + *record_count = 0; + + /* unpack logs from protobuf payload */ + if (encoder == NULL || options == NULL || options->metadata_key == NULL || + record_count == NULL || (payload == NULL && payload_size != 0)) { + return -1; + } + + input_logs = protobuf_logs_decode((uint8_t *) payload, payload_size, &protobuf_arena, + &arena_allocator, &protobuf_allocator); + if (input_logs == NULL) { + flb_warn("[opentelemetry] failed to unpack protobuf logs payload"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + resource_logs = input_logs->resource_logs; + if (input_logs->n_resource_logs == 0) { + ret = 0; + goto binary_payload_to_msgpack_end; + } + + if (resource_logs == NULL) { + flb_warn("[opentelemetry] no resource logs found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + for (resource_logs_index = 0; resource_logs_index < input_logs->n_resource_logs; resource_logs_index++) { + resource_log = resource_logs[resource_logs_index]; + if (resource_log == NULL) { + flb_warn("[opentelemetry] null resource logs entry found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + resource = resource_log->resource; + scope_logs = resource_log->scope_logs; + + if (resource_log->n_scope_logs > 0 && scope_logs == NULL) { + flb_warn("[opentelemetry] no scope logs found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + for (scope_log_index = 0; scope_log_index < resource_log->n_scope_logs; scope_log_index++) { + scope_log = scope_logs[scope_log_index]; + if (scope_log == NULL) { + flb_warn("[opentelemetry] null scope logs entry found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + log_records = scope_log->log_records; + + if (scope_log->n_log_records == 0) { + continue; + } + + if (log_records == NULL) { + flb_warn("[opentelemetry] no log records found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + flb_log_event_encoder_group_init(encoder); + + /* pack schema (internal) */ + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_STRING_VALUE("schema", 6), + FLB_LOG_EVENT_STRING_VALUE("otlp", 4), + FLB_LOG_EVENT_STRING_VALUE("resource_id", 11), + FLB_LOG_EVENT_INT64_VALUE(resource_logs_index), + FLB_LOG_EVENT_STRING_VALUE("scope_id", 8), + FLB_LOG_EVENT_INT64_VALUE(scope_log_index)); + + + ret = flb_log_event_encoder_dynamic_field_reset(&encoder->body); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[opentelemetry] failed to reset log event body: %s", + flb_log_event_encoder_get_error_description(ret)); + goto binary_payload_to_msgpack_end; + } + + flb_mp_map_header_init(&mh, mp_pck); + + /* Resource */ + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 8); + msgpack_pack_str_body(mp_pck, "resource", 8); + + flb_mp_map_header_init(&mh_tmp, mp_pck); + if (resource) { + /* look for OTel resource attributes */ + if (resource->n_attributes > 0 && resource->attributes) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "attributes", 10); + + ret = otel_pack_kvarray(mp_pck, + resource->attributes, + resource->n_attributes); + if (ret != 0) { + goto binary_payload_to_msgpack_end; + } + } + + if (resource->dropped_attributes_count > 0) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 24); + msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); + msgpack_pack_uint64(mp_pck, resource->dropped_attributes_count); + } + + if (resource_log->schema_url) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "schema_url", 10); + + len = strlen(resource_log->schema_url); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, resource_log->schema_url, len); + } + } + flb_mp_map_header_end(&mh_tmp); + + /* scope */ + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 5); + msgpack_pack_str_body(mp_pck, "scope", 5); + + /* Scope */ + scope = scope_log->scope; + scope_has_schema_url = FLB_FALSE; + + if (scope_log->schema_url && strlen(scope_log->schema_url) > 0) { + scope_has_schema_url = FLB_TRUE; + } + + if (scope && (scope->name || scope->version || + scope->n_attributes > 0 || scope->dropped_attributes_count > 0 || + scope_has_schema_url == FLB_TRUE)) { + flb_mp_map_header_init(&mh_tmp, mp_pck); + + if (scope_has_schema_url == FLB_TRUE) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "schema_url", 10); + + len = strlen(scope_log->schema_url); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, scope_log->schema_url, len); + } + + if (scope->name && strlen(scope->name) > 0) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 4); + msgpack_pack_str_body(mp_pck, "name", 4); + + len = strlen(scope->name); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, scope->name, len); + } + if (scope->version && strlen(scope->version) > 0) { + flb_mp_map_header_append(&mh_tmp); + + msgpack_pack_str(mp_pck, 7); + msgpack_pack_str_body(mp_pck, "version", 7); + + len = strlen(scope->version); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, scope->version, len); + } + + if (scope->n_attributes > 0 && scope->attributes) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "attributes", 10); + ret = otel_pack_kvarray(mp_pck, + scope->attributes, + scope->n_attributes); + if (ret != 0) { + goto binary_payload_to_msgpack_end; + } + } + + if (scope->dropped_attributes_count > 0) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 24); + msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); + msgpack_pack_uint64(mp_pck, scope->dropped_attributes_count); + } + + flb_mp_map_header_end(&mh_tmp); + } + else { + flb_mp_map_header_init(&mh_tmp, mp_pck); + + if (scope_has_schema_url == FLB_TRUE) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "schema_url", 10); + + len = strlen(scope_log->schema_url); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, scope_log->schema_url, len); + } + + flb_mp_map_header_end(&mh_tmp); + } + + flb_mp_map_header_end(&mh); + + ret = flb_log_event_encoder_dynamic_field_flush(&encoder->body); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[opentelemetry] could not set group content metadata: %s", + flb_log_event_encoder_get_error_description(ret)); + goto binary_payload_to_msgpack_end; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto binary_payload_to_msgpack_end; + } + + for (log_record_index=0; log_record_index < scope_log->n_log_records; log_record_index++) { + ret = flb_log_event_encoder_begin_record(encoder); + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + if (log_records[log_record_index]->time_unix_nano > 0) { + ret = flb_time_from_uint64( + &tm, + log_records[log_record_index]->time_unix_nano); + if (ret == 0) { + ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + } + } + else if (log_records[log_record_index]->observed_time_unix_nano > 0) { + ret = flb_time_from_uint64( + &tm, + log_records[log_record_index]->observed_time_unix_nano); + if (ret == 0) { + ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + } + } + else { + flb_time_get(&tm); + ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + } + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_dynamic_field_reset(&encoder->metadata); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[opentelemetry] failed to reset log event metadata: %s", + flb_log_event_encoder_get_error_description(ret)); + ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; + } + else { + ret = otel_pack_v1_metadata(options, + mp_pck_meta, + log_records[log_record_index], + resource, + scope_log->scope); + } + + if (ret != 0) { + flb_error("[opentelemetry] failed to convert log record"); + ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; + } + else { + ret = flb_log_event_encoder_dynamic_field_flush(&encoder->metadata); + } + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_dynamic_field_reset(&encoder->body); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[opentelemetry] failed to reset log event body: %s", + flb_log_event_encoder_get_error_description(ret)); + ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; + } + else if (options->body_key == NULL && + log_records[log_record_index]->body != NULL && + log_records[log_record_index]->body->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { + ret = otlp_pack_any_value( + mp_pck, + log_records[log_record_index]->body); + } + else { + logs_body_key = (char *) options->body_key; + if (logs_body_key == NULL) { + logs_body_key = "log"; + } + ret = msgpack_pack_map(mp_pck, 1); + if (ret == 0) { + ret = msgpack_pack_str(mp_pck, strlen(logs_body_key)); + } + if (ret == 0) { + ret = msgpack_pack_str_body(mp_pck, + logs_body_key, + strlen(logs_body_key)); + } + if (ret == 0) { + ret = otlp_pack_any_value( + mp_pck, + log_records[log_record_index]->body); + } + } + + if (ret != 0) { + flb_error("[opentelemetry] failed to convert log record body"); + ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; + } + else { + ret = flb_log_event_encoder_dynamic_field_flush(&encoder->body); + } + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_commit_record(encoder); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + (*record_count)++; + } + else { + goto binary_payload_to_msgpack_end; + } + } + else { + flb_error("[opentelemetry] marshalling error"); + goto binary_payload_to_msgpack_end; + } + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto binary_payload_to_msgpack_end; + } + + } + } + + binary_payload_to_msgpack_end: + if (input_logs) { + protobuf_logs_free(input_logs, protobuf_allocator); + } + cfl_arena_destroy(protobuf_arena); + + if (ret != 0) { + return -1; + } + + return 0; +} + +/* + * OTLP encoding functions to build the log records as CFL objects + * --------------------------------------------------------------- + * + * When the input processors can take a chunk of CFL records, the protobuf + * request is converted straight into the records the processors work on, + * skipping the msgpack buffer they would otherwise decode. The records match + * what the processors get from the msgpack encoding above: non-negative + * integers are unsigned values and strings or bytes are borrowed from the + * unpacked request, which outlives the processing. + * + * 'size' is the MessagePack size the msgpack encoding above produces for the + * same request. It enforces 'logs_max_size' exactly like that path, without + * building the intermediate MessagePack buffer: metadata and group maps use + * 32-bit headers, attribute maps, kvlists and the body map compact ones. + */ +#define OTEL_CFL_MAP_HEADER_SIZE 5 +#define OTEL_CFL_RECORD_SIZE 12 + +struct otel_cfl_builder { + const struct flb_opentelemetry_logs_decode_options *options; + struct flb_mp_chunk_cobj *chunk; + size_t size; + size_t limit; +}; + +static int otel_cfl_account(struct otel_cfl_builder *builder, size_t bytes) +{ + if (bytes > builder->limit || builder->size > builder->limit - bytes) { + return -1; + } + + builder->size += bytes; + return 0; +} + +static size_t otel_cfl_uint_size(uint64_t value) +{ + if (value <= 0x7f) { + return 1; + } + if (value <= UINT8_MAX) { + return 2; + } + if (value <= UINT16_MAX) { + return 3; + } + if (value <= UINT32_MAX) { + return 5; + } + + return 9; +} + +static size_t otel_cfl_int_size(int64_t value) +{ + if (value >= -32) { + return 1; + } + if (value >= INT8_MIN) { + return 2; + } + if (value >= INT16_MIN) { + return 3; + } + if (value >= INT32_MIN) { + return 5; + } + + return 9; +} + +static size_t otel_cfl_string_header_size(size_t length) +{ + if (length <= 31) { + return 1; + } + if (length <= UINT8_MAX) { + return 2; + } + if (length <= UINT16_MAX) { + return 3; + } + + return 5; +} + +static size_t otel_cfl_binary_header_size(size_t length) +{ + if (length <= UINT8_MAX) { + return 2; + } + if (length <= UINT16_MAX) { + return 3; + } + + return 5; +} + +static size_t otel_cfl_array_header_size(size_t entries) +{ + if (entries <= 15) { + return 1; + } + if (entries <= UINT16_MAX) { + return 3; + } + + return 5; +} + +static int otel_cfl_account_sized_value(struct otel_cfl_builder *builder, + size_t header_size, size_t value_size) +{ + if (otel_cfl_account(builder, header_size) != 0) { + return -1; + } + + return otel_cfl_account(builder, value_size); +} + +static struct cfl_variant *otel_cfl_string(struct otel_cfl_builder *builder, + char *str, size_t len) +{ + if (otel_cfl_account_sized_value(builder, + otel_cfl_string_header_size(len), len) != 0) { + return NULL; + } + + return cfl_variant_create_from_string_s(str, len, CFL_TRUE); +} + +static struct cfl_variant *otel_cfl_bytes(struct otel_cfl_builder *builder, + ProtobufCBinaryData bytes) +{ + if (otel_cfl_account_sized_value(builder, + otel_cfl_binary_header_size(bytes.len), + bytes.len) != 0) { + return NULL; + } + + return cfl_variant_create_from_bytes((char *) bytes.data, bytes.len, CFL_TRUE); +} + +static struct cfl_variant *otel_cfl_uint(struct otel_cfl_builder *builder, uint64_t val) +{ + if (otel_cfl_account(builder, otel_cfl_uint_size(val)) != 0) { + return NULL; + } + + return cfl_variant_create_from_uint64(val); +} + +/* msgpack keeps non-negative integers as unsigned values */ +static struct cfl_variant *otel_cfl_int(struct otel_cfl_builder *builder, int64_t val) +{ + if (val >= 0) { + return otel_cfl_uint(builder, (uint64_t) val); + } + + if (otel_cfl_account(builder, otel_cfl_int_size(val)) != 0) { + return NULL; + } + + return cfl_variant_create_from_int64(val); +} + +/* Insert a value, the value is released when it cannot be inserted */ +static int otel_cfl_insert(struct otel_cfl_builder *builder, struct cfl_kvlist *kvlist, + const char *key, size_t key_len, struct cfl_variant *value) +{ + if (value == NULL) { + return -1; + } + + if (otel_cfl_account_sized_value(builder, + otel_cfl_string_header_size(key_len), + key_len) != 0 || + cfl_kvlist_insert_s(kvlist, (char *) key, key_len, value) != 0) { + cfl_variant_destroy(value); + return -1; + } + + return 0; +} + +/* Insert a kvlist, the kvlist is released when it cannot be inserted */ +static int otel_cfl_insert_kvlist(struct otel_cfl_builder *builder, + struct cfl_kvlist *kvlist, + const char *key, size_t key_len, + struct cfl_kvlist *value) +{ + struct cfl_variant *variant; + + if (value == NULL) { + return -1; + } + + variant = cfl_variant_create_from_kvlist(value); + if (variant == NULL) { + cfl_kvlist_destroy(value); + return -1; + } + + return otel_cfl_insert(builder, kvlist, key, key_len, variant); +} + +static struct cfl_kvlist *otel_cfl_kvlist_create(struct otel_cfl_builder *builder) +{ + /* CFL's dynamic map encoder reserves a map32 header for every kvlist. */ + if (otel_cfl_account(builder, OTEL_CFL_MAP_HEADER_SIZE) != 0) { + return NULL; + } + + return cfl_kvlist_create(); +} + +/* A map the msgpack path packs with a compact header */ +static struct cfl_kvlist *otel_cfl_kvlist_create_compact(struct otel_cfl_builder *builder, + size_t entries) +{ + if (otel_cfl_account(builder, otel_cfl_array_header_size(entries)) != 0) { + return NULL; + } + + return cfl_kvlist_create(); +} + +static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__AnyValue *value); + +static struct cfl_kvlist *otel_cfl_kvarray(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__KeyValue **kv_array, + size_t kv_count) +{ + size_t index; + struct cfl_kvlist *kvlist; + + kvlist = otel_cfl_kvlist_create_compact(builder, kv_count); + if (kvlist == NULL) { + return NULL; + } + + for (index = 0; index < kv_count; index++) { + if (otel_cfl_insert(builder, kvlist, + kv_array[index]->key, strlen(kv_array[index]->key), + otel_cfl_any_value(builder, kv_array[index]->value)) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + } + + return kvlist; +} + +static struct cfl_variant *otel_cfl_array(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__ArrayValue *array) +{ + size_t index; + struct cfl_array *entries; + struct cfl_variant *entry; + struct cfl_variant *variant; + + if (otel_cfl_account(builder, + otel_cfl_array_header_size(array->n_values)) != 0) { + return NULL; + } + + entries = cfl_array_create(array->n_values); + if (entries == NULL) { + return NULL; + } + + for (index = 0; index < array->n_values; index++) { + entry = otel_cfl_any_value(builder, array->values[index]); + if (entry == NULL || cfl_array_append(entries, entry) != 0) { + if (entry != NULL) { + cfl_variant_destroy(entry); + } + cfl_array_destroy(entries); + return NULL; + } + } + + variant = cfl_variant_create_from_array(entries); + if (variant == NULL) { + cfl_array_destroy(entries); + } + + return variant; +} + +static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__AnyValue *value) +{ + struct cfl_kvlist *kvlist; + struct cfl_variant *variant; + + if (value == NULL) { + if (otel_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_null(); + } + + switch (value->value_case) { + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: + return otel_cfl_string(builder, value->string_value, strlen(value->string_value)); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: + if (otel_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_bool(value->bool_value); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: + return otel_cfl_int(builder, value->int_value); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: + if (otel_cfl_account(builder, 9) != 0) { + return NULL; + } + return cfl_variant_create_from_double(value->double_value); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: + return otel_cfl_array(builder, value->array_value); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: + kvlist = otel_cfl_kvarray(builder, value->kvlist_value->values, + value->kvlist_value->n_values); + if (kvlist == NULL) { + return NULL; + } + + variant = cfl_variant_create_from_kvlist(kvlist); + if (variant == NULL) { + cfl_kvlist_destroy(kvlist); + } + return variant; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: + return otel_cfl_bytes(builder, value->bytes_value); + + /* profiling-only string dictionary references are ignored in logs */ + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE_STRINDEX: + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET: + if (otel_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_null(); + + default: + flb_error("[otel]: invalid value type in pack_any_value"); + return NULL; + } +} + +/* Wrap a kvlist as a record object, the kvlist is released on failure */ +static struct cfl_object *otel_cfl_object(struct cfl_kvlist *kvlist) +{ + struct cfl_object *object; + + if (kvlist == NULL) { + return NULL; + } + + object = cfl_object_create(); + if (object == NULL) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + + object->variant = cfl_variant_create_from_kvlist(kvlist); + if (object->variant == NULL) { + cfl_kvlist_destroy(kvlist); + cfl_object_destroy(object); + return NULL; + } + object->type = CFL_OBJECT_KVLIST; + + return object; +} + +/* Append a record to the chunk, its objects are owned by the chunk */ +static struct flb_mp_chunk_record *otel_cfl_record_add(struct otel_cfl_builder *builder, + struct flb_time *tm) +{ + struct flb_mp_chunk_record *record; + + if (otel_cfl_account(builder, OTEL_CFL_RECORD_SIZE) != 0) { + return NULL; + } + + record = flb_mp_chunk_record_create(builder->chunk); + if (record == NULL) { + return NULL; + } + + memset(&record->event, 0, sizeof(record->event)); + flb_time_copy(&record->event.timestamp, tm); + cfl_list_add(&record->_head, &builder->chunk->records); + + return record; +} + +/* https://opentelemetry.io/docs/specs/otel/logs/data-model/#log-and-event-record-definition */ +static struct cfl_kvlist *otel_cfl_v1_metadata( + struct otel_cfl_builder *builder, + Opentelemetry__Proto__Logs__V1__LogRecord *log_record) +{ + int ret = 0; + struct cfl_kvlist *otlp; + struct cfl_kvlist *metadata; + + metadata = otel_cfl_kvlist_create(builder); + otlp = otel_cfl_kvlist_create(builder); + if (metadata == NULL || otlp == NULL) { + goto error; + } + + if (log_record->observed_time_unix_nano != 0) { + ret = otel_cfl_insert(builder, otlp, "observed_timestamp", 18, + otel_cfl_uint(builder, log_record->observed_time_unix_nano)); + } + + /* Value of 0 indicates unknown or missing timestamp. */ + if (ret == 0 && log_record->time_unix_nano != 0) { + ret = otel_cfl_insert(builder, otlp, "timestamp", 9, + otel_cfl_uint(builder, log_record->time_unix_nano)); + } + + /* https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber */ + if (ret == 0 && log_record->severity_number >= 1 && log_record->severity_number <= 24) { + ret = otel_cfl_insert(builder, otlp, "severity_number", 15, + otel_cfl_uint(builder, log_record->severity_number)); + } + + if (ret == 0 && log_record->severity_text != NULL && + strlen(log_record->severity_text) > 0) { + ret = otel_cfl_insert(builder, otlp, "severity_text", 13, + otel_cfl_string(builder, log_record->severity_text, + strlen(log_record->severity_text))); + } + + if (ret == 0 && log_record->n_attributes > 0) { + ret = otel_cfl_insert_kvlist(builder, otlp, "attributes", 10, + otel_cfl_kvarray(builder, log_record->attributes, + log_record->n_attributes)); + } + + if (ret == 0 && log_record->dropped_attributes_count > 0) { + ret = otel_cfl_insert(builder, otlp, "dropped_attributes_count", 24, + otel_cfl_uint(builder, log_record->dropped_attributes_count)); + } + + if (ret == 0 && log_record->trace_id.len > 0) { + ret = otel_cfl_insert(builder, otlp, "trace_id", 8, + otel_cfl_bytes(builder, log_record->trace_id)); + } + + if (ret == 0 && log_record->span_id.len > 0) { + ret = otel_cfl_insert(builder, otlp, "span_id", 7, + otel_cfl_bytes(builder, log_record->span_id)); + } + + if (ret == 0) { + ret = otel_cfl_insert(builder, otlp, "trace_flags", 11, + otel_cfl_uint(builder, log_record->flags & 0xff)); + } + + if (ret == 0 && log_record->event_name != NULL && strlen(log_record->event_name) > 0) { + ret = otel_cfl_insert(builder, otlp, "event_name", 10, + otel_cfl_string(builder, log_record->event_name, + strlen(log_record->event_name))); + } + + if (ret != 0) { + goto error; + } + + ret = otel_cfl_insert_kvlist(builder, metadata, builder->options->metadata_key, + strlen(builder->options->metadata_key), otlp); + otlp = NULL; + if (ret != 0) { + goto error; + } + + return metadata; + +error: + if (otlp != NULL) { + cfl_kvlist_destroy(otlp); + } + if (metadata != NULL) { + cfl_kvlist_destroy(metadata); + } + return NULL; +} + +static struct cfl_kvlist *otel_cfl_body(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__AnyValue *body) +{ + const char *logs_body_key; + struct cfl_kvlist *kvlist; + + if (builder->options->body_key == NULL && body != NULL && + body->value_case == OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { + return otel_cfl_kvarray(builder, body->kvlist_value->values, + body->kvlist_value->n_values); + } + + logs_body_key = builder->options->body_key; + if (logs_body_key == NULL) { + logs_body_key = "log"; + } + + kvlist = otel_cfl_kvlist_create_compact(builder, 1); + if (kvlist == NULL) { + return NULL; + } + + if (otel_cfl_insert(builder, kvlist, logs_body_key, strlen(logs_body_key), + otel_cfl_any_value(builder, body)) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + + return kvlist; +} + +static struct cfl_kvlist *otel_cfl_group_metadata(struct otel_cfl_builder *builder, + int resource_index, int scope_index) +{ + struct cfl_kvlist *kvlist; + + kvlist = otel_cfl_kvlist_create(builder); + if (kvlist == NULL) { + return NULL; + } + + if (otel_cfl_insert(builder, kvlist, "schema", 6, otel_cfl_string(builder, "otlp", 4)) != 0 || + otel_cfl_insert(builder, kvlist, "resource_id", 11, + otel_cfl_int(builder, resource_index)) != 0 || + otel_cfl_insert(builder, kvlist, "scope_id", 8, + otel_cfl_int(builder, scope_index)) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + + return kvlist; +} + +static struct cfl_kvlist *otel_cfl_group_body( + struct otel_cfl_builder *builder, + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log, + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log) +{ + int ret = 0; + struct cfl_kvlist *body; + struct cfl_kvlist *resource_kv; + struct cfl_kvlist *scope_kv; + Opentelemetry__Proto__Resource__V1__Resource *resource; + Opentelemetry__Proto__Common__V1__InstrumentationScope *scope; + + body = otel_cfl_kvlist_create(builder); + resource_kv = otel_cfl_kvlist_create(builder); + scope_kv = otel_cfl_kvlist_create(builder); + if (body == NULL || resource_kv == NULL || scope_kv == NULL) { + goto error; + } + + resource = resource_log->resource; + if (resource) { + /* look for OTel resource attributes */ + if (resource->n_attributes > 0 && resource->attributes) { + ret = otel_cfl_insert_kvlist(builder, resource_kv, "attributes", 10, + otel_cfl_kvarray(builder, resource->attributes, + resource->n_attributes)); + } + + if (ret == 0 && resource->dropped_attributes_count > 0) { + ret = otel_cfl_insert(builder, resource_kv, "dropped_attributes_count", 24, + otel_cfl_uint(builder, resource->dropped_attributes_count)); + } + + if (ret == 0 && resource_log->schema_url) { + ret = otel_cfl_insert(builder, resource_kv, "schema_url", 10, + otel_cfl_string(builder, resource_log->schema_url, + strlen(resource_log->schema_url))); + } + } + + scope = scope_log->scope; + if (ret == 0 && scope_log->schema_url && strlen(scope_log->schema_url) > 0) { + ret = otel_cfl_insert(builder, scope_kv, "schema_url", 10, + otel_cfl_string(builder, scope_log->schema_url, + strlen(scope_log->schema_url))); + } + + if (scope) { + if (ret == 0 && scope->name && strlen(scope->name) > 0) { + ret = otel_cfl_insert(builder, scope_kv, "name", 4, + otel_cfl_string(builder, scope->name, strlen(scope->name))); + } + + if (ret == 0 && scope->version && strlen(scope->version) > 0) { + ret = otel_cfl_insert(builder, scope_kv, "version", 7, + otel_cfl_string(builder, scope->version, + strlen(scope->version))); + } + + if (ret == 0 && scope->n_attributes > 0 && scope->attributes) { + ret = otel_cfl_insert_kvlist(builder, scope_kv, "attributes", 10, + otel_cfl_kvarray(builder, scope->attributes, + scope->n_attributes)); + } + + if (ret == 0 && scope->dropped_attributes_count > 0) { + ret = otel_cfl_insert(builder, scope_kv, "dropped_attributes_count", 24, + otel_cfl_uint(builder, scope->dropped_attributes_count)); + } + } + + if (ret != 0) { + goto error; + } + + ret = otel_cfl_insert_kvlist(builder, body, "resource", 8, resource_kv); + resource_kv = NULL; + if (ret == 0) { + ret = otel_cfl_insert_kvlist(builder, body, "scope", 5, scope_kv); + scope_kv = NULL; + } + if (ret != 0) { + goto error; + } + + return body; + +error: + if (scope_kv != NULL) { + cfl_kvlist_destroy(scope_kv); + } + if (resource_kv != NULL) { + cfl_kvlist_destroy(resource_kv); + } + if (body != NULL) { + cfl_kvlist_destroy(body); + } + return NULL; +} + +/* Build the group start, records and group end of one scope logs entry */ +static int otel_cfl_scope_logs(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log, + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log, + int resource_index, int scope_index, + size_t *record_count) +{ + size_t index; + struct flb_time tm; + struct flb_mp_chunk_record *group; + struct flb_mp_chunk_record *record; + Opentelemetry__Proto__Logs__V1__LogRecord *log_record; + + flb_time_set(&tm, FLB_LOG_EVENT_GROUP_START, 0); + group = otel_cfl_record_add(builder, &tm); + if (group == NULL) { + return -1; + } + + group->cobj_metadata = otel_cfl_object( + otel_cfl_group_metadata(builder, resource_index, scope_index)); + group->cobj_record = otel_cfl_object(otel_cfl_group_body(builder, resource_log, scope_log)); + if (group->cobj_metadata == NULL || group->cobj_record == NULL) { + return -1; + } + + /* the group start owns the group content, like a decoded chunk */ + group->cobj_group_metadata = group->cobj_metadata; + group->cobj_group_attributes = group->cobj_record; + group->owns_group_metadata = FLB_TRUE; + group->owns_group_attributes = FLB_TRUE; + + for (index = 0; index < scope_log->n_log_records; index++) { + log_record = scope_log->log_records[index]; + + if (log_record->time_unix_nano > 0) { + if (flb_time_from_uint64(&tm, log_record->time_unix_nano) != 0) { + return -1; + } + } + else if (log_record->observed_time_unix_nano > 0) { + if (flb_time_from_uint64(&tm, log_record->observed_time_unix_nano) != 0) { + return -1; + } + } + else { + flb_time_get(&tm); + } + + record = otel_cfl_record_add(builder, &tm); + if (record == NULL) { + return -1; + } + + record->cobj_metadata = otel_cfl_object(otel_cfl_v1_metadata(builder, log_record)); + if (record->cobj_metadata == NULL) { + flb_error("[opentelemetry] failed to convert log record"); + return -1; + } + + record->cobj_record = otel_cfl_object(otel_cfl_body(builder, log_record->body)); + if (record->cobj_record == NULL) { + flb_error("[opentelemetry] failed to convert log record body"); + return -1; + } + + record->cobj_group_metadata = group->cobj_group_metadata; + record->cobj_group_attributes = group->cobj_group_attributes; + (*record_count)++; + } + + flb_time_set(&tm, FLB_LOG_EVENT_GROUP_END, 0); + record = otel_cfl_record_add(builder, &tm); + if (record == NULL) { + return -1; + } + + record->cobj_metadata = otel_cfl_object(otel_cfl_kvlist_create(builder)); + record->cobj_record = otel_cfl_object(otel_cfl_kvlist_create(builder)); + if (record->cobj_metadata == NULL || record->cobj_record == NULL) { + return -1; + } + record->cobj_group_metadata = group->cobj_group_metadata; + record->cobj_group_attributes = group->cobj_group_attributes; + + return 0; +} + +/* + * CFL records borrow strings and byte arrays from the unpacked protobuf + * request. Keep the request and its arena alive until the caller consumes the + * chunk synchronously. + */ +struct flb_opentelemetry_logs_protobuf_cfl { + struct flb_mp_chunk_cobj *chunk; + size_t encoded_size; + size_t record_count; + struct cfl_arena *protobuf_arena; + ProtobufCAllocator arena_allocator; + ProtobufCAllocator *protobuf_allocator; + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *input_logs; +}; + +int flb_opentelemetry_logs_protobuf_to_cfl( + const void *payload, size_t payload_size, + const struct flb_opentelemetry_logs_decode_options *options, + struct flb_opentelemetry_logs_protobuf_cfl **out_result) +{ + int ret; + size_t resource_logs_index; + size_t scope_log_index; + struct otel_cfl_builder builder; + struct flb_opentelemetry_logs_protobuf_cfl *result; + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; + + if (options == NULL || options->metadata_key == NULL || out_result == NULL || + (payload == NULL && payload_size != 0)) { + return -1; + } + + *out_result = NULL; + result = flb_calloc(1, sizeof(struct flb_opentelemetry_logs_protobuf_cfl)); + if (result == NULL) { + flb_errno(); + return -1; + } + + builder.options = options; + builder.size = 0; + builder.limit = options->output_limit; + builder.chunk = NULL; + ret = 0; + + result->input_logs = protobuf_logs_decode( + (uint8_t *) payload, payload_size, + &result->protobuf_arena, + &result->arena_allocator, + &result->protobuf_allocator); + if (result->input_logs == NULL) { + flb_warn("[opentelemetry] failed to unpack protobuf logs payload"); + ret = -1; + goto end; + } + + if (result->input_logs->n_resource_logs > 0 && + result->input_logs->resource_logs == NULL) { + flb_warn("[opentelemetry] no resource logs found"); + ret = -1; + goto end; + } + + builder.chunk = flb_mp_chunk_cobj_create(NULL, NULL); + if (builder.chunk == NULL) { + ret = -1; + goto end; + } + builder.chunk->record_pool_enabled = FLB_TRUE; + + for (resource_logs_index = 0; + resource_logs_index < result->input_logs->n_resource_logs; + resource_logs_index++) { + resource_log = result->input_logs->resource_logs[resource_logs_index]; + if (resource_log == NULL) { + flb_warn("[opentelemetry] null resource logs entry found"); + ret = -1; + goto end; + } + + if (resource_log->n_scope_logs > 0 && resource_log->scope_logs == NULL) { + flb_warn("[opentelemetry] no scope logs found"); + ret = -1; + goto end; + } + + for (scope_log_index = 0; scope_log_index < resource_log->n_scope_logs; + scope_log_index++) { + scope_log = resource_log->scope_logs[scope_log_index]; + if (scope_log == NULL) { + flb_warn("[opentelemetry] null scope logs entry found"); + ret = -1; + goto end; + } + if (scope_log->n_log_records == 0) { + continue; + } + if (scope_log->log_records == NULL) { + flb_warn("[opentelemetry] no log records found"); + ret = -1; + goto end; + } + + ret = otel_cfl_scope_logs(&builder, resource_log, scope_log, + resource_logs_index, scope_log_index, + &result->record_count); + if (ret != 0) { + flb_error("[opentelemetry] marshalling error"); + goto end; + } + } + } + + result->encoded_size = builder.size; + result->chunk = builder.chunk; + builder.chunk = NULL; + *out_result = result; + +end: + if (builder.chunk != NULL) { + flb_mp_chunk_cobj_destroy(builder.chunk); + } + if (ret != 0) { + flb_opentelemetry_logs_protobuf_cfl_destroy(result); + return -1; + } + + return 0; +} + +struct flb_mp_chunk_cobj *flb_opentelemetry_logs_protobuf_cfl_take_chunk( + struct flb_opentelemetry_logs_protobuf_cfl *result) +{ + struct flb_mp_chunk_cobj *chunk; + + if (result == NULL) { + return NULL; + } + + chunk = result->chunk; + result->chunk = NULL; + + return chunk; +} + +size_t flb_opentelemetry_logs_protobuf_cfl_get_encoded_size( + struct flb_opentelemetry_logs_protobuf_cfl *result) +{ + if (result == NULL) { + return 0; + } + + return result->encoded_size; +} + +void flb_opentelemetry_logs_protobuf_cfl_destroy( + struct flb_opentelemetry_logs_protobuf_cfl *result) +{ + if (result == NULL) { + return; + } + + if (result->chunk != NULL) { + flb_mp_chunk_cobj_destroy(result->chunk); + } + if (result->input_logs != NULL) { + protobuf_logs_free(result->input_logs, result->protobuf_allocator); + } + cfl_arena_destroy(result->protobuf_arena); + flb_free(result); +} diff --git a/src/opentelemetry/flb_opentelemetry_protobuf.c b/src/opentelemetry/flb_opentelemetry_protobuf.c new file mode 100644 index 00000000000..736df7819f4 --- /dev/null +++ b/src/opentelemetry/flb_opentelemetry_protobuf.c @@ -0,0 +1,134 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +/* Includes the export request and intermediate protobuf message wrappers. */ +#define OTEL_PROTOBUF_MAX_DEPTH 100 + +struct protobuf_frame { + const ProtobufCMessageDescriptor *descriptor; + const unsigned char *cursor; + const unsigned char *end; +}; + +static int read_varint(struct protobuf_frame *frame, uint64_t *value) +{ + unsigned int shift; + unsigned char byte; + + *value = 0; + for (shift = 0; shift < 64; shift += 7) { + if (frame->cursor == frame->end) { + return -1; + } + byte = *frame->cursor++; + if (shift == 63 && byte > 1) { + return -1; + } + *value |= (uint64_t) (byte & 0x7f) << shift; + if ((byte & 0x80) == 0) { + return 0; + } + } + return -1; +} + +/* + * Inspect known message fields before protobuf-c recursively allocates them. + * Strings, bytes, packed scalars and unknown fields are opaque. The explicit + * stack bounds our own stack usage as well as the subsequent unpack operation. + */ +int flb_opentelemetry_protobuf_validate(const ProtobufCMessageDescriptor *descriptor, + const void *data, size_t size) +{ + struct protobuf_frame frames[OTEL_PROTOBUF_MAX_DEPTH]; + struct protobuf_frame *frame; + const ProtobufCFieldDescriptor *field; + const unsigned char *message; + uint64_t tag; + uint64_t length; + size_t depth; + + if (descriptor == NULL || (data == NULL && size != 0)) { + return -1; + } + if (size == 0) { + return 0; + } + + depth = 1; + frames[0].descriptor = descriptor; + frames[0].cursor = data; + frames[0].end = frames[0].cursor + size; + + while (depth > 0) { + frame = &frames[depth - 1]; + if (frame->cursor == frame->end) { + depth--; + continue; + } + if (read_varint(frame, &tag) != 0 || tag >> 3 == 0 || tag >> 3 > 0x1fffffff) { + return -1; + } + + switch (tag & 7) { + case 0: + if (read_varint(frame, &length) != 0) { + return -1; + } + continue; + case 1: + length = 8; + break; + case 2: + if (read_varint(frame, &length) != 0) { + return -1; + } + break; + case 5: + length = 4; + break; + default: + /* Groups are unsupported by protobuf-c. */ + return -1; + } + + if (length > (uint64_t) (frame->end - frame->cursor)) { + return -1; + } + message = frame->cursor; + frame->cursor += (size_t) length; + if ((tag & 7) != 2) { + continue; + } + field = protobuf_c_message_descriptor_get_field(frame->descriptor, tag >> 3); + if (field == NULL || field->type != PROTOBUF_C_TYPE_MESSAGE) { + continue; + } + if (depth >= OTEL_PROTOBUF_MAX_DEPTH) { + return -1; + } + frames[depth].descriptor = field->descriptor; + frames[depth].cursor = message; + frames[depth].end = message + (size_t) length; + depth++; + } + return 0; +} From 33a12fe798bda5417b522df2708950aaefd68944 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 21:32:35 -0600 Subject: [PATCH 08/33] tests: internal: use shared protobuf validator Signed-off-by: Eduardo Silva --- tests/internal/opentelemetry_protobuf.c | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/internal/opentelemetry_protobuf.c b/tests/internal/opentelemetry_protobuf.c index b8e1d4607dd..7b89cbee64d 100644 --- a/tests/internal/opentelemetry_protobuf.c +++ b/tests/internal/opentelemetry_protobuf.c @@ -1,5 +1,5 @@ #include "flb_tests_internal.h" -#include "../../plugins/in_opentelemetry/opentelemetry_protobuf.h" +#include static void test_protobuf_depth(void) { @@ -26,11 +26,11 @@ static void test_protobuf_depth(void) /* 99, 100 and 101 schema messages, including scalar AnyValue leaves. */ size = opentelemetry__proto__common__v1__any_value__pack(&values[1], buffer); - TEST_CHECK(opentelemetry_protobuf_validate(values[1].base.descriptor, buffer, size) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(values[1].base.descriptor, buffer, size) == 0); size = opentelemetry__proto__common__v1__array_value__pack(&arrays[0], buffer); - TEST_CHECK(opentelemetry_protobuf_validate(arrays[0].base.descriptor, buffer, size) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(arrays[0].base.descriptor, buffer, size) == 0); size = opentelemetry__proto__common__v1__any_value__pack(&values[0], buffer); - TEST_CHECK(opentelemetry_protobuf_validate(values[0].base.descriptor, buffer, size) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(values[0].base.descriptor, buffer, size) != 0); } static void test_protobuf_wire_boundaries(void) @@ -45,14 +45,18 @@ static void test_protobuf_wire_boundaries(void) unsigned char fixed[] = {0x21, 0, 0, 0, 0, 0, 0, 0, 0}; descriptor = &opentelemetry__proto__common__v1__any_value__descriptor; - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, NULL, 0) == 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, truncated, sizeof(truncated)) != 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, overflow, sizeof(overflow)) != 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, opaque, sizeof(opaque)) == 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, unknown, sizeof(unknown)) == 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, zero_tag, sizeof(zero_tag)) != 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, fixed, sizeof(fixed)) == 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, fixed, sizeof(fixed) - 1) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, NULL, 0) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, truncated, + sizeof(truncated)) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, overflow, + sizeof(overflow)) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, opaque, sizeof(opaque)) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, unknown, sizeof(unknown)) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, zero_tag, + sizeof(zero_tag)) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, fixed, sizeof(fixed)) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, fixed, + sizeof(fixed) - 1) != 0); } TEST_LIST = { From 2d504adcc4edf9066e07a5082e0acaa48a0f44d2 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 21:32:35 -0600 Subject: [PATCH 09/33] in_opentelemetry: use shared protobuf log decoding Decode protobuf logs with the shared OTLP decoder. When every logs processor of the input is native, and worker safe for HTTP workers, the request is built directly as CFL records and processed in the worker that received it, skipping the msgpack encoding the processors would decode. Other chains keep the msgpack path. Like the OpenTelemetry Collector, a request without log records is accepted instead of being rejected. Signed-off-by: Eduardo Silva --- plugins/in_opentelemetry/CMakeLists.txt | 1 - plugins/in_opentelemetry/opentelemetry_logs.c | 824 ++---------------- .../in_opentelemetry/opentelemetry_protobuf.c | 131 --- .../in_opentelemetry/opentelemetry_protobuf.h | 31 - 4 files changed, 54 insertions(+), 933 deletions(-) delete mode 100644 plugins/in_opentelemetry/opentelemetry_protobuf.c delete mode 100644 plugins/in_opentelemetry/opentelemetry_protobuf.h diff --git a/plugins/in_opentelemetry/CMakeLists.txt b/plugins/in_opentelemetry/CMakeLists.txt index 543d9efd13f..5b31b406906 100644 --- a/plugins/in_opentelemetry/CMakeLists.txt +++ b/plugins/in_opentelemetry/CMakeLists.txt @@ -4,7 +4,6 @@ endif() set(src opentelemetry.c - opentelemetry_protobuf.c opentelemetry_prot.c opentelemetry_logs.c opentelemetry_traces.c diff --git a/plugins/in_opentelemetry/opentelemetry_logs.c b/plugins/in_opentelemetry/opentelemetry_logs.c index 52dc92dab38..9e6792b8116 100644 --- a/plugins/in_opentelemetry/opentelemetry_logs.c +++ b/plugins/in_opentelemetry/opentelemetry_logs.c @@ -20,777 +20,15 @@ #include #include #include -#include #include -#include +#include +#include #include -#include -#include - -#include "opentelemetry_protobuf.h" #include "opentelemetry.h" #include "opentelemetry_logs.h" #include "opentelemetry_utils.h" -/* - * Small requests use protobuf-c's default allocator to avoid reserving an - * arena chunk. Realistic batched requests use the arena to amortize the large - * number of short-lived allocations performed by protobuf-c. - */ -#define OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE 8192 -#define OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE 4096 -#define OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE 65536 - -static void *protobuf_arena_chunk_malloc(void *context, size_t size) -{ - (void) context; - - return flb_malloc(size); -} - -static void protobuf_arena_chunk_free(void *context, void *pointer) -{ - (void) context; - - flb_free(pointer); -} - -static void *protobuf_arena_alloc(void *allocator_data, size_t size) -{ - struct cfl_arena *arena; - - arena = allocator_data; - if (size == 0) { - size = 1; - } - - return cfl_arena_malloc(arena, size); -} - -static void protobuf_arena_free(void *allocator_data, void *pointer) -{ - (void) allocator_data; - (void) pointer; -} - -static struct cfl_arena *protobuf_arena_create(void) -{ - struct cfl_arena_options options; - - cfl_arena_options_init(&options); - options.chunk_size = OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE; - options.maximum_chunk_size = OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE; - options.malloc_fn = protobuf_arena_chunk_malloc; - options.free_fn = protobuf_arena_chunk_free; - - return cfl_arena_create_with_options(&options); -} - -static Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest * -protobuf_logs_unpack(ProtobufCAllocator *allocator, size_t size, const uint8_t *data) -{ - if (opentelemetry_protobuf_validate( - &opentelemetry__proto__collector__logs__v1__export_logs_service_request__descriptor, - data, size) != 0) { - return NULL; - } - return opentelemetry__proto__collector__logs__v1__export_logs_service_request__unpack( - allocator, size, data); -} - -static void protobuf_logs_free( - Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *logs, - ProtobufCAllocator *allocator) -{ - opentelemetry__proto__collector__logs__v1__export_logs_service_request__free_unpacked( - logs, allocator); -} - -/* - * OTLP encoding functions to pack the log records as msgpack - * ---------------------------------------------------------- - */ -static int otlp_pack_any_value(msgpack_packer *mp_pck, Opentelemetry__Proto__Common__V1__AnyValue *body); - -static int otel_pack_string(msgpack_packer *mp_pck, char *str) -{ - return msgpack_pack_str_with_body(mp_pck, str, strlen(str)); -} - -static int otel_pack_bool(msgpack_packer *mp_pck, bool val) -{ - if (val) { - return msgpack_pack_true(mp_pck); - } - else { - return msgpack_pack_false(mp_pck); - } -} - -static int otel_pack_int(msgpack_packer *mp_pck, int64_t val) -{ - return msgpack_pack_int64(mp_pck, val); -} - -static int otel_pack_double(msgpack_packer *mp_pck, double val) -{ - return msgpack_pack_double(mp_pck, val); -} - -static int otel_pack_kvarray(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__KeyValue **kv_array, - size_t kv_count) -{ - int result; - int index; - - result = msgpack_pack_map(mp_pck, kv_count); - - if (result != 0) { - return result; - } - - for (index = 0; index < kv_count && result == 0; index++) { - result = otel_pack_string(mp_pck, kv_array[index]->key); - - if(result == 0) { - result = otlp_pack_any_value(mp_pck, kv_array[index]->value); - } - } - - return result; -} - -static int otel_pack_kvlist(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__KeyValueList *kv_list) -{ - int kv_index; - int ret; - char *key; - Opentelemetry__Proto__Common__V1__AnyValue *value; - - ret = msgpack_pack_map(mp_pck, kv_list->n_values); - if (ret != 0) { - return ret; - } - - for (kv_index = 0; kv_index < kv_list->n_values && ret == 0; kv_index++) { - key = kv_list->values[kv_index]->key; - value = kv_list->values[kv_index]->value; - - ret = otel_pack_string(mp_pck, key); - - if(ret == 0) { - ret = otlp_pack_any_value(mp_pck, value); - } - } - - return ret; -} - -static int otel_pack_array(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__ArrayValue *array) -{ - int ret; - int array_index; - - ret = msgpack_pack_array(mp_pck, array->n_values); - - if (ret != 0) { - return ret; - } - - for (array_index = 0; array_index < array->n_values && ret == 0; array_index++) { - ret = otlp_pack_any_value(mp_pck, array->values[array_index]); - } - - return ret; -} - -static int otel_pack_bytes(msgpack_packer *mp_pck, - ProtobufCBinaryData bytes) -{ - return msgpack_pack_bin_with_body(mp_pck, bytes.data, bytes.len); -} - -static int otlp_pack_any_value(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__AnyValue *body) -{ - int result; - - result = -2; - - if (body == NULL) { - msgpack_pack_nil(mp_pck); - return 0; - } - - switch(body->value_case){ - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: - result = otel_pack_string(mp_pck, body->string_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE_STRINDEX: - /* Profiling-only string dictionary reference: ignore in logs. */ - result = msgpack_pack_nil(mp_pck); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: - result = otel_pack_bool(mp_pck, body->bool_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: - result = otel_pack_int(mp_pck, body->int_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: - result = otel_pack_double(mp_pck, body->double_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: - result = otel_pack_array(mp_pck, body->array_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: - result = otel_pack_kvlist(mp_pck, body->kvlist_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: - result = otel_pack_bytes(mp_pck, body->bytes_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET: - /* treat an unset value as null */ - result = msgpack_pack_nil(mp_pck); - break; - - default: - break; - } - - if (result == -2) { - flb_error("[otel]: invalid value type in pack_any_value"); - result = -1; - } - - return result; -} - -/* https://opentelemetry.io/docs/specs/otel/logs/data-model/#log-and-event-record-definition */ -static int otel_pack_v1_metadata(struct flb_opentelemetry *ctx, - msgpack_packer *mp_pck, - struct Opentelemetry__Proto__Logs__V1__LogRecord *log_record, - Opentelemetry__Proto__Resource__V1__Resource *resource, - Opentelemetry__Proto__Common__V1__InstrumentationScope *scope) -{ - int ret; - int len; - struct flb_mp_map_header mh; - struct flb_mp_map_header otlp_mh; - - flb_mp_map_header_init(&otlp_mh, mp_pck); - - len = flb_sds_len(ctx->logs_metadata_key); - - /* otlp key start */ - flb_mp_map_header_append(&otlp_mh); - - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, ctx->logs_metadata_key, len); - - flb_mp_map_header_init(&mh, mp_pck); - - if (log_record->observed_time_unix_nano != 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 18); - msgpack_pack_str_body(mp_pck, "observed_timestamp", 18); - msgpack_pack_uint64(mp_pck, log_record->observed_time_unix_nano); - } - - /* Value of 0 indicates unknown or missing timestamp. */ - if (log_record->time_unix_nano != 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "timestamp", 9); - msgpack_pack_uint64(mp_pck, log_record->time_unix_nano); - } - - /* https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber */ - if (log_record->severity_number >= 1 && log_record->severity_number <= 24) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 15); - msgpack_pack_str_body(mp_pck, "severity_number", 15); - msgpack_pack_uint64(mp_pck, log_record->severity_number); - } - - if (log_record->severity_text != NULL && strlen(log_record->severity_text) > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 13); - msgpack_pack_str_body(mp_pck, "severity_text", 13); - msgpack_pack_str(mp_pck, strlen(log_record->severity_text)); - msgpack_pack_str_body(mp_pck, log_record->severity_text, strlen(log_record->severity_text)); - } - - if (log_record->n_attributes > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "attributes", 10); - ret = otel_pack_kvarray(mp_pck, - log_record->attributes, - log_record->n_attributes); - if (ret != 0) { - return ret; - } - } - - if (log_record->dropped_attributes_count > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 24); - msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); - msgpack_pack_uint64(mp_pck, log_record->dropped_attributes_count); - } - - if (log_record->trace_id.len > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 8); - msgpack_pack_str_body(mp_pck, "trace_id", 8); - ret = otel_pack_bytes(mp_pck, log_record->trace_id); - if (ret != 0) { - return ret; - } - } - - if (log_record->span_id.len > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "span_id", 7); - ret = otel_pack_bytes(mp_pck, log_record->span_id); - if (ret != 0) { - return ret; - } - } - - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 11); - msgpack_pack_str_body(mp_pck, "trace_flags", 11); - msgpack_pack_uint8(mp_pck, (uint8_t) log_record->flags & 0xff); - - if (log_record->event_name != NULL && strlen(log_record->event_name) > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "event_name", 10); - msgpack_pack_str(mp_pck, strlen(log_record->event_name)); - msgpack_pack_str_body(mp_pck, log_record->event_name, strlen(log_record->event_name)); - } - - flb_mp_map_header_end(&mh); - - /* otlp key end */ - flb_mp_map_header_end(&otlp_mh); - - return 0; -} - -static int binary_payload_to_msgpack(struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - char *tag, size_t tag_len, - uint8_t *in_buf, - size_t in_size, - size_t *record_count) -{ - int ret = 0; - int len; - int resource_logs_index; - int scope_log_index; - int log_record_index; - char *logs_body_key; - int scope_has_schema_url; - struct cfl_arena *protobuf_arena; - struct flb_mp_map_header mh; - struct flb_mp_map_header mh_tmp; - struct flb_time tm; - ProtobufCAllocator arena_allocator; - ProtobufCAllocator *protobuf_allocator; - - msgpack_packer *mp_pck; - msgpack_packer *mp_pck_meta; - - /* OTel proto suff */ - Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *input_logs; - Opentelemetry__Proto__Logs__V1__ScopeLogs **scope_logs; - Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; - Opentelemetry__Proto__Common__V1__InstrumentationScope *scope; - - Opentelemetry__Proto__Logs__V1__ResourceLogs **resource_logs; - Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; - Opentelemetry__Proto__Logs__V1__LogRecord **log_records; - Opentelemetry__Proto__Resource__V1__Resource *resource; - - mp_pck = &encoder->body.packer; - mp_pck_meta = &encoder->metadata.packer; - input_logs = NULL; - protobuf_arena = NULL; - protobuf_allocator = NULL; - *record_count = 0; - - if (in_size >= OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE) { - protobuf_arena = protobuf_arena_create(); - if (protobuf_arena != NULL) { - arena_allocator.alloc = protobuf_arena_alloc; - arena_allocator.free = protobuf_arena_free; - arena_allocator.allocator_data = protobuf_arena; - protobuf_allocator = &arena_allocator; - } - } - - /* unpack logs from protobuf payload */ - input_logs = protobuf_logs_unpack(protobuf_allocator, in_size, in_buf); - if (input_logs == NULL) { - flb_plg_warn(ctx->ins, "failed to unpack input logs from OpenTelemetry payload"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - resource_logs = input_logs->resource_logs; - if (input_logs->n_resource_logs == 0) { - ret = 0; - goto binary_payload_to_msgpack_end; - } - - if (resource_logs == NULL) { - flb_plg_warn(ctx->ins, "no resource logs found"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - for (resource_logs_index = 0; resource_logs_index < input_logs->n_resource_logs; resource_logs_index++) { - resource_log = resource_logs[resource_logs_index]; - if (resource_log == NULL) { - flb_plg_warn(ctx->ins, "null resource logs entry found"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - resource = resource_log->resource; - scope_logs = resource_log->scope_logs; - - if (resource_log->n_scope_logs > 0 && scope_logs == NULL) { - flb_plg_warn(ctx->ins, "no scope logs found"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - for (scope_log_index = 0; scope_log_index < resource_log->n_scope_logs; scope_log_index++) { - scope_log = scope_logs[scope_log_index]; - if (scope_log == NULL) { - flb_plg_warn(ctx->ins, "null scope logs entry found"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - log_records = scope_log->log_records; - - if (scope_log->n_log_records == 0) { - continue; - } - - if (log_records == NULL) { - flb_plg_warn(ctx->ins, "no log records found"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - flb_log_event_encoder_group_init(encoder); - - /* pack schema (internal) */ - ret = flb_log_event_encoder_append_metadata_values(encoder, - FLB_LOG_EVENT_STRING_VALUE("schema", 6), - FLB_LOG_EVENT_STRING_VALUE("otlp", 4), - FLB_LOG_EVENT_STRING_VALUE("resource_id", 11), - FLB_LOG_EVENT_INT64_VALUE(resource_logs_index), - FLB_LOG_EVENT_STRING_VALUE("scope_id", 8), - FLB_LOG_EVENT_INT64_VALUE(scope_log_index)); - - - ret = flb_log_event_encoder_dynamic_field_reset(&encoder->body); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "failed to reset log event body: %s", - flb_log_event_encoder_get_error_description(ret)); - goto binary_payload_to_msgpack_end; - } - - flb_mp_map_header_init(&mh, mp_pck); - - /* Resource */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 8); - msgpack_pack_str_body(mp_pck, "resource", 8); - - flb_mp_map_header_init(&mh_tmp, mp_pck); - if (resource) { - /* look for OTel resource attributes */ - if (resource->n_attributes > 0 && resource->attributes) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "attributes", 10); - - ret = otel_pack_kvarray(mp_pck, - resource->attributes, - resource->n_attributes); - if (ret != 0) { - goto binary_payload_to_msgpack_end; - } - } - - if (resource->dropped_attributes_count > 0) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 24); - msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); - msgpack_pack_uint64(mp_pck, resource->dropped_attributes_count); - } - - if (resource_log->schema_url) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "schema_url", 10); - - len = strlen(resource_log->schema_url); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, resource_log->schema_url, len); - } - } - flb_mp_map_header_end(&mh_tmp); - - /* scope */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 5); - msgpack_pack_str_body(mp_pck, "scope", 5); - - /* Scope */ - scope = scope_log->scope; - scope_has_schema_url = FLB_FALSE; - - if (scope_log->schema_url && strlen(scope_log->schema_url) > 0) { - scope_has_schema_url = FLB_TRUE; - } - - if (scope && (scope->name || scope->version || - scope->n_attributes > 0 || scope->dropped_attributes_count > 0 || - scope_has_schema_url == FLB_TRUE)) { - flb_mp_map_header_init(&mh_tmp, mp_pck); - - if (scope_has_schema_url == FLB_TRUE) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "schema_url", 10); - - len = strlen(scope_log->schema_url); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, scope_log->schema_url, len); - } - - if (scope->name && strlen(scope->name) > 0) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 4); - msgpack_pack_str_body(mp_pck, "name", 4); - - len = strlen(scope->name); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, scope->name, len); - } - if (scope->version && strlen(scope->version) > 0) { - flb_mp_map_header_append(&mh_tmp); - - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "version", 7); - - len = strlen(scope->version); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, scope->version, len); - } - - if (scope->n_attributes > 0 && scope->attributes) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "attributes", 10); - ret = otel_pack_kvarray(mp_pck, - scope->attributes, - scope->n_attributes); - if (ret != 0) { - goto binary_payload_to_msgpack_end; - } - } - - if (scope->dropped_attributes_count > 0) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 24); - msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); - msgpack_pack_uint64(mp_pck, scope->dropped_attributes_count); - } - - flb_mp_map_header_end(&mh_tmp); - } - else { - flb_mp_map_header_init(&mh_tmp, mp_pck); - - if (scope_has_schema_url == FLB_TRUE) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "schema_url", 10); - - len = strlen(scope_log->schema_url); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, scope_log->schema_url, len); - } - - flb_mp_map_header_end(&mh_tmp); - } - - flb_mp_map_header_end(&mh); - - ret = flb_log_event_encoder_dynamic_field_flush(&encoder->body); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "could not set group content metadata: %s", - flb_log_event_encoder_get_error_description(ret)); - goto binary_payload_to_msgpack_end; - } - - ret = flb_log_event_encoder_group_header_end(encoder); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - goto binary_payload_to_msgpack_end; - } - - for (log_record_index=0; log_record_index < scope_log->n_log_records; log_record_index++) { - ret = flb_log_event_encoder_begin_record(encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (log_records[log_record_index]->time_unix_nano > 0) { - ret = flb_time_from_uint64( - &tm, - log_records[log_record_index]->time_unix_nano); - if (ret == 0) { - ret = flb_log_event_encoder_set_timestamp(encoder, &tm); - } - } - else if (log_records[log_record_index]->observed_time_unix_nano > 0) { - ret = flb_time_from_uint64( - &tm, - log_records[log_record_index]->observed_time_unix_nano); - if (ret == 0) { - ret = flb_log_event_encoder_set_timestamp(encoder, &tm); - } - } - else { - flb_time_get(&tm); - ret = flb_log_event_encoder_set_timestamp(encoder, &tm); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_dynamic_field_reset(&encoder->metadata); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "failed to reset log event metadata: %s", - flb_log_event_encoder_get_error_description(ret)); - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - ret = otel_pack_v1_metadata(ctx, - mp_pck_meta, - log_records[log_record_index], - resource, - scope_log->scope); - } - - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to convert log record"); - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - ret = flb_log_event_encoder_dynamic_field_flush(&encoder->metadata); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_dynamic_field_reset(&encoder->body); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "failed to reset log event body: %s", - flb_log_event_encoder_get_error_description(ret)); - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else if (ctx->logs_body_key == NULL && - log_records[log_record_index]->body != NULL && - log_records[log_record_index]->body->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { - ret = otlp_pack_any_value( - mp_pck, - log_records[log_record_index]->body); - } - else { - logs_body_key = ctx->logs_body_key; - if (logs_body_key == NULL) { - logs_body_key = "log"; - } - ret = msgpack_pack_map(mp_pck, 1); - if (ret == 0) { - ret = msgpack_pack_str(mp_pck, strlen(logs_body_key)); - } - if (ret == 0) { - ret = msgpack_pack_str_body(mp_pck, - logs_body_key, - strlen(logs_body_key)); - } - if (ret == 0) { - ret = otlp_pack_any_value( - mp_pck, - log_records[log_record_index]->body); - } - } - - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to convert log record body"); - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - ret = flb_log_event_encoder_dynamic_field_flush(&encoder->body); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(encoder); - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - (*record_count)++; - } - else { - goto binary_payload_to_msgpack_end; - } - } - else { - flb_plg_error(ctx->ins, "marshalling error"); - goto binary_payload_to_msgpack_end; - } - } - - ret = flb_log_event_encoder_group_end(encoder); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - goto binary_payload_to_msgpack_end; - } - - } - } - - binary_payload_to_msgpack_end: - if (input_logs) { - protobuf_logs_free(input_logs, protobuf_allocator); - } - cfl_arena_destroy(protobuf_arena); - - if (ret != 0) { - return -1; - } - - return 0; -} /* * Main function used from opentelemetry_prot.c to process logs either in JSON or Protobuf format. @@ -822,11 +60,20 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, uint64_t payload_size; size_t record_count; struct flb_log_event_encoder *encoder; + struct flb_mp_chunk_cobj *chunk; + struct flb_opentelemetry_logs_decode_options decode_options; + struct flb_opentelemetry_logs_protobuf_cfl *cfl_result; + int worker_thread; buf = (char *) data; payload = data; payload_size = size; record_count = 0; + cfl_result = NULL; + worker_thread = opentelemetry_uses_worker_ingress_queue(ctx); + decode_options.metadata_key = ctx->logs_metadata_key; + decode_options.body_key = ctx->logs_body_key; + decode_options.output_limit = *remaining; /* Detect the type of payload */ if (content_type) { @@ -846,6 +93,41 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, } } + /* + * Protobuf records go straight to the input processors as CFL records when + * they can take them. Worker threads enqueue the serialized processor + * output as already processed, so the engine does not run the chain twice. + */ + if (is_proto == FLB_TRUE && + flb_input_log_chunk_supported(ctx->ins, worker_thread) == FLB_TRUE) { + ret = flb_opentelemetry_logs_protobuf_to_cfl( + payload, payload_size, &decode_options, &cfl_result); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to process logs from protobuf payload"); + return -1; + } + + *remaining -= flb_opentelemetry_logs_protobuf_cfl_get_encoded_size(cfl_result); + chunk = flb_opentelemetry_logs_protobuf_cfl_take_chunk(cfl_result); + + /* like the Collector, a request without log records is accepted */ + if (cfl_list_is_empty(&chunk->records)) { + flb_mp_chunk_cobj_destroy(chunk); + ret = 0; + } + else { + ret = flb_input_log_append_chunk(ctx->ins, tag, flb_sds_len(tag), + chunk, worker_thread); + } + flb_opentelemetry_logs_protobuf_cfl_destroy(cfl_result); + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to append logs to the input buffer"); + return -1; + } + + return 0; + } + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); if (encoder == NULL) { return -1; @@ -853,10 +135,8 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, encoder->output_limit = *remaining; if (is_proto == FLB_TRUE) { - ret = binary_payload_to_msgpack(ctx, encoder, - tag, tag_len, - (uint8_t *) payload, payload_size, - &record_count); + ret = flb_opentelemetry_logs_protobuf_to_msgpack( + encoder, payload, payload_size, &decode_options, &record_count); if (ret < 0) { flb_plg_error(ctx->ins, "failed to process logs from protobuf payload"); } @@ -875,9 +155,13 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, } - if (ret >= 0) { + if (ret >= 0 && encoder->output_length == 0) { + /* like the Collector, a request without log records is accepted */ + ret = 0; + } + else if (ret >= 0) { *remaining -= encoder->output_length; - if (opentelemetry_uses_worker_ingress_queue(ctx)) { + if (worker_thread == FLB_TRUE) { size_t allocation_size; void *resized_buffer; diff --git a/plugins/in_opentelemetry/opentelemetry_protobuf.c b/plugins/in_opentelemetry/opentelemetry_protobuf.c deleted file mode 100644 index ff302ddc3b0..00000000000 --- a/plugins/in_opentelemetry/opentelemetry_protobuf.c +++ /dev/null @@ -1,131 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "opentelemetry_protobuf.h" - -struct protobuf_frame { - const ProtobufCMessageDescriptor *descriptor; - const unsigned char *cursor; - const unsigned char *end; -}; - -static int read_varint(struct protobuf_frame *frame, uint64_t *value) -{ - unsigned int shift; - unsigned char byte; - - *value = 0; - for (shift = 0; shift < 64; shift += 7) { - if (frame->cursor == frame->end) { - return -1; - } - byte = *frame->cursor++; - if (shift == 63 && byte > 1) { - return -1; - } - *value |= (uint64_t) (byte & 0x7f) << shift; - if ((byte & 0x80) == 0) { - return 0; - } - } - return -1; -} - -/* - * Inspect known message fields before protobuf-c recursively allocates them. - * Strings, bytes, packed scalars and unknown fields are opaque. The explicit - * stack bounds our own stack usage as well as the subsequent unpack operation. - */ -int opentelemetry_protobuf_validate(const ProtobufCMessageDescriptor *descriptor, - const void *data, size_t size) -{ - struct protobuf_frame frames[OTEL_PROTOBUF_MAX_DEPTH]; - struct protobuf_frame *frame; - const ProtobufCFieldDescriptor *field; - const unsigned char *message; - uint64_t tag; - uint64_t length; - size_t depth; - - if (descriptor == NULL || (data == NULL && size != 0)) { - return -1; - } - if (size == 0) { - return 0; - } - - depth = 1; - frames[0].descriptor = descriptor; - frames[0].cursor = data; - frames[0].end = frames[0].cursor + size; - - while (depth > 0) { - frame = &frames[depth - 1]; - if (frame->cursor == frame->end) { - depth--; - continue; - } - if (read_varint(frame, &tag) != 0 || tag >> 3 == 0 || tag >> 3 > 0x1fffffff) { - return -1; - } - - switch (tag & 7) { - case 0: - if (read_varint(frame, &length) != 0) { - return -1; - } - continue; - case 1: - length = 8; - break; - case 2: - if (read_varint(frame, &length) != 0) { - return -1; - } - break; - case 5: - length = 4; - break; - default: - /* Groups are unsupported by protobuf-c. */ - return -1; - } - - if (length > (uint64_t) (frame->end - frame->cursor)) { - return -1; - } - message = frame->cursor; - frame->cursor += (size_t) length; - if ((tag & 7) != 2) { - continue; - } - field = protobuf_c_message_descriptor_get_field(frame->descriptor, tag >> 3); - if (field == NULL || field->type != PROTOBUF_C_TYPE_MESSAGE) { - continue; - } - if (depth >= OTEL_PROTOBUF_MAX_DEPTH) { - return -1; - } - frames[depth].descriptor = field->descriptor; - frames[depth].cursor = message; - frames[depth].end = message + (size_t) length; - depth++; - } - return 0; -} diff --git a/plugins/in_opentelemetry/opentelemetry_protobuf.h b/plugins/in_opentelemetry/opentelemetry_protobuf.h deleted file mode 100644 index 5f3e39ede59..00000000000 --- a/plugins/in_opentelemetry/opentelemetry_protobuf.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_OPENTELEMETRY_PROTOBUF_H -#define FLB_IN_OPENTELEMETRY_PROTOBUF_H - -#include - -/* Includes the export request and intermediate protobuf message wrappers. */ -#define OTEL_PROTOBUF_MAX_DEPTH 100 - -int opentelemetry_protobuf_validate(const ProtobufCMessageDescriptor *descriptor, - const void *data, size_t size); - -#endif From 279c31d62cf9545205db8fad0c36f921ed1464ce Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 21:32:35 -0600 Subject: [PATCH 10/33] tests: internal: cover CFL processor execution Signed-off-by: Eduardo Silva --- tests/internal/processor.c | 151 +++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/tests/internal/processor.c b/tests/internal/processor.c index 91824e7f3e3..d0f1135be00 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -693,6 +693,7 @@ static void processor_metrics_failure_clears_output(void) #define WINDOW_PLAIN_RECORDS 2000 #define WINDOW_TOTAL_RECORDS (WINDOW_GROUPS * WINDOW_GROUP_RECORDS + \ 2 * WINDOW_PLAIN_RECORDS) +#define WINDOW_CORRUPT_RECORDS 3000 static int window_calls; static int window_max_records; @@ -792,6 +793,54 @@ static int create_window_records(char **out_buf, size_t *out_size) return 0; } +static int create_plain_window_records(char **out_buf, size_t *out_size) +{ + int i; + int ret = 0; + struct flb_log_event_encoder *encoder; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + for (i = 0; ret == 0 && i < WINDOW_CORRUPT_RECORDS; i++) { + ret = append_window_record(encoder, i); + } + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + *out_buf = flb_malloc(encoder->output_length + 1); + if (*out_buf == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memcpy(*out_buf, encoder->output_buffer, encoder->output_length); + (*out_buf)[encoder->output_length] = (char) 0xc1; + *out_size = encoder->output_length + 1; + flb_log_event_encoder_destroy(encoder); + + return 0; +} + +static int append_corrupt_tail(char **buf, size_t *size) +{ + char *tmp; + + tmp = flb_realloc(*buf, *size + 1); + if (tmp == NULL) { + return -1; + } + + tmp[*size] = (char) 0xc1; + *buf = tmp; + *size += 1; + return 0; +} + static int mark_window_records(struct flb_processor_instance *ins, void *data, const char *tag, int tag_len) { @@ -1107,6 +1156,105 @@ static void processor_log_windows_drop_all(void) window_config_destroy(config); } +static void check_marked_output(void *output, size_t output_size, int expected) +{ + int count = 0; + int type; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == + FLB_EVENT_DECODER_SUCCESS); + + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type != FLB_LOG_EVENT_NORMAL) { + continue; + } + TEST_CHECK(event.body->type == MSGPACK_OBJECT_MAP); + TEST_CHECK(event.body->via.map.size == 3); + count++; + } + + TEST_CHECK(count == expected); + flb_log_event_decoder_destroy(&decoder); +} + +static void run_processor_corrupt_tail(int windowed) +{ + int ret; + int records = -1; + int expected; + char *input; + size_t input_size; + void *output = NULL; + size_t output_size = 0; + double errors; + char stage_label[32]; + struct flb_config *config; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_input_instance owner; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + + if (windowed) { + TEST_ASSERT(create_plain_window_records(&input, &input_size) == 0); + expected = WINDOW_CORRUPT_RECORDS; + proc = window_processor_create(config, &owner, "test_window_mark", NULL); + } + else { + TEST_ASSERT(create_msgpack_records(&input, &input_size) == 0); + TEST_ASSERT(append_corrupt_tail(&input, &input_size) == 0); + expected = 1; + proc = window_processor_create(config, &owner, "test_single_mark", NULL); + } + TEST_ASSERT(proc != NULL); + + window_calls = 0; + ret = flb_processor_run_with_count(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &output, &output_size, + -1, &records); + TEST_CHECK(ret == 0); + TEST_CHECK(records == expected); + TEST_CHECK(output != NULL && output != input); + TEST_CHECK(flb_mp_count_log_records(output, output_size) == expected); + if (windowed) { + TEST_CHECK(window_calls > 1); + } + else { + TEST_CHECK(window_calls == 1); + } + check_marked_output(output, output_size, expected); + + pu = mk_list_entry(proc->logs.next, struct flb_processor_unit, _head); + snprintf(stage_label, sizeof(stage_label), "%zu", pu->stage); + TEST_CHECK(get_counter_value_5(proc->cmt_errors, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", &errors) == 0); + TEST_CHECK(errors == 1.0); + + flb_free(output); + flb_free(input); + flb_processor_destroy(proc); + cmt_destroy(owner.cmt); + window_config_destroy(config); +} + +static void processor_corrupt_tail_preserves_prefix(void) +{ + run_processor_corrupt_tail(FLB_FALSE); +} + +static void processor_window_corrupt_tail_preserves_prefix(void) +{ + run_processor_corrupt_tail(FLB_TRUE); +} + static void processor_log_windows_need_every_unit(void) { int i; @@ -1562,6 +1710,9 @@ TEST_LIST = { { "processor_metrics_counters", processor_metrics_counters }, { "processor_log_windows_match_single_chunk", processor_log_windows_match_single_chunk }, { "processor_log_windows_drop_all", processor_log_windows_drop_all }, + { "processor_corrupt_tail_preserves_prefix", processor_corrupt_tail_preserves_prefix }, + { "processor_window_corrupt_tail_preserves_prefix", + processor_window_corrupt_tail_preserves_prefix }, { "processor_log_windows_need_every_unit", processor_log_windows_need_every_unit }, { "processor_chunk_codec_is_per_run", processor_chunk_codec_is_per_run }, { "processor_group_boundaries_follow_the_input", processor_group_boundaries_follow_the_input }, From ef661aea3c6ea9fd8dd804d99c28e5a4a497d00e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 21:32:35 -0600 Subject: [PATCH 11/33] tests: integration: cover OTLP worker processing Signed-off-by: Eduardo Silva --- .../tests/test_in_opentelemetry_001.py | 484 +++++++++++++++++- .../config/out_stdout_processor_workers.yaml | 48 ++ .../out_stdout/tests/test_out_stdout_001.py | 21 + 3 files changed, 552 insertions(+), 1 deletion(-) create mode 100644 tests/integration/scenarios/out_stdout/config/out_stdout_processor_workers.yaml diff --git a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py index bd368d88654..20fb3bdbf7b 100644 --- a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py +++ b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py @@ -520,7 +520,8 @@ def generated_limit_service(tmp_path, workers, limit="16k"): (32, 64, 1, 1, False), (64, 2, 1, 2, False), (1, 1, 512, 1, False), - (64, 1000, 0, 1, False), + # scopes without records produce nothing, the request is accepted + (64, 1000, 0, 1, True), ]) def test_in_opentelemetry_generated_limit(tmp_path, workers, encoding, attributes, scopes, records, resources, accepted): @@ -2696,3 +2697,484 @@ def test_in_opentelemetry_http_snappy_invalid_checksum(frame_type): service.wait_for_signal_count("logs", 1) finally: service.stop() + + +# Processor chains used to compare the msgpack and the direct CFL logs paths. +# The direct path is only taken when the chain starts with a native processor, +# the filter first chain covers the fallback to the msgpack path. +# +# The msgpack reference runs the same chain behind a filter that keeps every +# record: a filter unit makes the input take the established msgpack path. +MSGPACK_REFERENCE_FILTER = {"name": "grep", "exclude": "msgpack_reference ^never$"} + +DIRECT_CFL_PROCESSOR_CHAINS = { + "content_modifier": [ + {"name": "content_modifier", "context": "otel_resource_attributes", + "action": "upsert", "key": "aaa", "value": "bbb"}, + {"name": "content_modifier", "context": "otel_resource_attributes", + "action": "delete", "key": "service.name"}, + {"name": "content_modifier", "context": "otel_scope_name", + "action": "upsert", "value": "new scope name"}, + {"name": "content_modifier", "context": "attributes", + "action": "upsert", "key": "added", "value": "yes"}, + ], + "sql_drop": [ + {"name": "sql", "query": "SELECT * FROM STREAM WHERE keep = 'yes';"}, + {"name": "content_modifier", "context": "otel_scope_version", + "action": "upsert", "value": "3.1.0"}, + ], + "native_then_filter": [ + {"name": "content_modifier", "context": "otel_scope_attributes", + "action": "upsert", "key": "scope_new", "value": "123"}, + {"name": "grep", "regex": "keep ^yes$"}, + ], + "filter_first": [ + {"name": "grep", "regex": "keep ^yes$"}, + {"name": "content_modifier", "context": "otel_resource_attributes", + "action": "upsert", "key": "aaa", "value": "bbb"}, + ], + "hash_once": [ + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + ], + "hash_then_sql": [ + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + {"name": "sql", "query": "SELECT * FROM STREAM WHERE keep = 'yes';"}, + ], + "conditional_safe": [ + {"name": "content_modifier", "context": "attributes", + "action": "upsert", "key": "condition_matched", "value": "yes", + "condition": {"op": "and", "rules": [ + {"field": "$keep", "op": "eq", "value": "yes"} + ]}}, + {"name": "content_modifier", "context": "attributes", + "action": "upsert", "key": "always", "value": "yes"}, + ], + "hash_then_filter": [ + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + {"name": "grep", "regex": "keep ^yes$"}, + ], + "hash_then_unflagged": [ + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + {"name": "opentelemetry_envelope"}, + ], + "unflagged_then_hash": [ + {"name": "opentelemetry_envelope"}, + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + ], +} + + +def add_any_value_attributes(attributes): + attributes.add(key="str").value.string_value = "value" + attributes.add(key="int_negative").value.int_value = -(2**40) + attributes.add(key="int_positive").value.int_value = 2**62 + attributes.add(key="double").value.double_value = 3.25 + attributes.add(key="bool").value.bool_value = True + attributes.add(key="bytes").value.bytes_value = b"\x00\x01binary\xff" + attributes.add(key="empty") + array = attributes.add(key="array").value.array_value + array.values.add().string_value = "one" + array.values.add().int_value = 2 + array.values.add() + nested = array.values.add().kvlist_value + nested.values.add(key="inner").value.bool_value = False + kvlist = attributes.add(key="kvlist").value.kvlist_value + kvlist.values.add(key="name").value.string_value = "nested" + kvlist.values.add(key="list").value.array_value.values.add().double_value = -1.5 + + +def add_kvlist_body(record, keep, marker): + body = record.body.kvlist_value + body.values.add(key="keep").value.string_value = keep + body.values.add(key="marker").value.string_value = marker + body.values.add(key="count").value.int_value = -5 + add_any_value_attributes(body.values.add(key="nested").value.kvlist_value.values) + + +def build_direct_cfl_logs_payload(): + request = ExportLogsServiceRequest() + timestamp = 1789516800123456789 + + resource_logs = request.resource_logs.add(schema_url="https://example.com/resource") + resource_logs.resource.attributes.add(key="service.name").value.string_value = "direct" + add_any_value_attributes(resource_logs.resource.attributes) + resource_logs.resource.dropped_attributes_count = 2 + + scope_logs = resource_logs.scope_logs.add(schema_url="https://example.com/scope") + scope_logs.scope.name = "direct-cfl" + scope_logs.scope.version = "1.0.0" + scope_logs.scope.dropped_attributes_count = 1 + add_any_value_attributes(scope_logs.scope.attributes) + + record = scope_logs.log_records.add(time_unix_nano=timestamp, + observed_time_unix_nano=timestamp + 1, + severity_number=9, severity_text="INFO", + dropped_attributes_count=3, flags=0x101, + trace_id=bytes(range(16)), span_id=bytes(range(8))) + add_kvlist_body(record, "yes", "kvlist") + add_any_value_attributes(record.attributes) + + scope_logs.log_records.add(time_unix_nano=timestamp + 2).body.string_value = "plain" + scope_logs.log_records.add(observed_time_unix_nano=timestamp + 3).body.int_value = -7 + scope_logs.log_records.add(time_unix_nano=timestamp + 4, + severity_number=24).body.bytes_value = b"\xde\xad" + array = scope_logs.log_records.add(time_unix_nano=timestamp + 5).body.array_value + array.values.add().double_value = 0.5 + array.values.add().bool_value = False + scope_logs.log_records.add(time_unix_nano=timestamp + 6, severity_number=99) + + # scopes without records produce no group + resource_logs.scope_logs.add().scope.name = "empty" + + # resources and scopes without content + second = request.resource_logs.add() + second_scope = second.scope_logs.add() + add_kvlist_body(second_scope.log_records.add(time_unix_nano=timestamp + 7), "yes", "bare") + add_kvlist_body(second_scope.log_records.add(time_unix_nano=timestamp + 8), "no", "bare") + second.scope_logs.add(schema_url="https://example.com/only-schema") + add_kvlist_body(second.scope_logs[1].log_records.add(time_unix_nano=timestamp + 9), + "yes", "schema-only") + + return request + + +def build_direct_cfl_sentinel_payload(keep="yes"): + request = ExportLogsServiceRequest() + record = request.resource_logs.add().scope_logs.add().log_records.add( + time_unix_nano=1789516800000000000) + add_kvlist_body(record, keep, "sentinel") + return request + + +def direct_cfl_service(tmp_path, direct, processors, *, body_key=None, workers=1, + logs_max_size=None, threaded=False, conditional_routes=False): + name = f"direct-cfl-{'on' if direct else 'off'}.yaml" + if not direct: + processors = [MSGPACK_REFERENCE_FILTER] + (processors or []) + options = [f"http_server.workers: {workers}"] + if body_key: + options.append(f"logs_body_key: {body_key}") + if logs_max_size: + options.append(f"logs_max_size: {logs_max_size}") + if threaded: + options.append("threaded: true") + option_lines = "\n".join(f" {option}" for option in options) + route_lines = "" + output_match = " match: '*'" + if conditional_routes: + route_lines = """ routes: + logs: + - name: direct_cfl_default + condition: + default: true + to: + outputs: + - direct_cfl_output +""" + output_match = "" + config = tmp_path / name + config.write_text(f""" +service: + flush: 0.2 + grace: 1 + log_level: info + http_server: on + http_port: ${{FLUENT_BIT_HTTP_MONITORING_PORT}} +pipeline: + inputs: + - name: opentelemetry + port: ${{FLUENT_BIT_TEST_LISTENER_PORT}} + http2: on +{option_lines} + processors: {json.dumps({"logs": processors})} +{route_lines} outputs: + - name: opentelemetry + alias: direct_cfl_output +{output_match} + host: 127.0.0.1 + port: ${{TEST_SUITE_HTTP_PORT}} +""", encoding="utf-8") + return Service(str(config)) + + +def flatten_received_logs(requests_received): + records = [] + for request in requests_received: + for resource_logs in request.resource_logs: + resource = json_format.MessageToDict(resource_logs.resource) + for scope_logs in resource_logs.scope_logs: + scope = json_format.MessageToDict(scope_logs.scope) + for record in scope_logs.log_records: + records.append(json.dumps({ + "resource": resource, + "resource_schema_url": resource_logs.schema_url, + "scope": scope, + "scope_schema_url": scope_logs.schema_url, + "record": json_format.MessageToDict(record), + }, sort_keys=True)) + return sorted(records) + + +def is_sentinel(record): + return '"stringValue": "sentinel"' in record + + +def run_direct_cfl_case(tmp_path, direct, processors, transport, *, body_key=None, + workers=1, sentinel_keep="yes", threaded=False, + conditional_routes=False, encoding="protobuf"): + service = direct_cfl_service(tmp_path, direct, processors, body_key=body_key, + workers=workers, threaded=threaded, + conditional_routes=conditional_routes) + service.start() + try: + before = len(data_storage["logs"]) + payloads = [build_direct_cfl_logs_payload(), + build_direct_cfl_sentinel_payload(sentinel_keep)] + for payload in payloads: + if transport == "grpc": + service.send_grpc_request("logs", payload) + elif encoding == "json": + # protobuf's generic JSON formatter represents bytes as base64, + # while OTLP/JSON requires trace and span IDs in hexadecimal. + for resource_logs in payload.resource_logs: + for scope_logs in resource_logs.scope_logs: + for record in scope_logs.log_records: + record.trace_id = b"" + record.span_id = b"" + json_payload = json_format.MessageToJson(payload).encode("utf-8") + response = service.send_raw_request( + "/v1/logs", json_payload, content_type="application/json") + assert response.status_code == 201 + else: + response = service.send_raw_request("/v1/logs", payload.SerializeToString()) + assert response.status_code == 201 + + # The sentinel goes in its own request, after the payload being compared. + def received(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if any(is_sentinel(record) for record in records) else None + + return service.service.wait_for_condition(received, timeout=20, interval=0.1, + description="direct CFL sentinel") + finally: + service.stop() + + +@pytest.mark.parametrize("transport", ["http", "grpc"]) +@pytest.mark.parametrize("chain", sorted(DIRECT_CFL_PROCESSOR_CHAINS)) +def test_in_opentelemetry_logs_direct_cfl_parity(tmp_path, chain, transport): + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + msgpack_records = run_direct_cfl_case(tmp_path, False, processors, transport) + direct_records = run_direct_cfl_case(tmp_path, True, processors, transport) + + # more than the sentinel must reach the output for the comparison to matter + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +def test_in_opentelemetry_logs_direct_cfl_body_key(tmp_path): + processors = DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"] + msgpack_records = run_direct_cfl_case(tmp_path, False, processors, "http", + body_key="message") + direct_records = run_direct_cfl_case(tmp_path, True, processors, "http", + body_key="message") + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +def test_in_opentelemetry_logs_direct_cfl_workers(tmp_path): + # Hashing makes accidental processing in both the worker and collector + # observable: a double pass would hash the first digest again. + processors = DIRECT_CFL_PROCESSOR_CHAINS["hash_once"] + msgpack_records = run_direct_cfl_case(tmp_path, False, processors, "http", workers=4) + direct_records = run_direct_cfl_case(tmp_path, True, processors, "http", workers=4) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("conditional_routes", [False, True]) +def test_in_opentelemetry_logs_direct_cfl_threaded_workers(tmp_path, conditional_routes): + processors = DIRECT_CFL_PROCESSOR_CHAINS["hash_once"] + msgpack_records = run_direct_cfl_case( + tmp_path, False, processors, "http", workers=4, threaded=True, + conditional_routes=conditional_routes) + direct_records = run_direct_cfl_case( + tmp_path, True, processors, "http", workers=4, threaded=True, + conditional_routes=conditional_routes) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("conditional_routes", [False, True]) +def test_in_opentelemetry_logs_worker_processing(tmp_path, encoding, conditional_routes): + # Worker-safe chains run in the HTTP workers, the msgpack reference runs + # them in the input collector. + processors = DIRECT_CFL_PROCESSOR_CHAINS["hash_once"] + collector_records = run_direct_cfl_case( + tmp_path, False, processors, "http", workers=4, threaded=True, + conditional_routes=conditional_routes, encoding=encoding) + worker_records = run_direct_cfl_case( + tmp_path, True, processors, "http", workers=4, threaded=True, + conditional_routes=conditional_routes, encoding=encoding) + + assert len(collector_records) > 1 + assert worker_records == collector_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("chain", ["hash_then_filter", "hash_then_unflagged", + "unflagged_then_hash"]) +def test_in_opentelemetry_logs_worker_processing_fallback(tmp_path, chain): + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + collector_records = run_direct_cfl_case( + tmp_path, False, processors, "http", workers=4, threaded=True, + conditional_routes=True) + fallback_records = run_direct_cfl_case( + tmp_path, True, processors, "http", workers=4, threaded=True, + conditional_routes=True) + + assert len(collector_records) > 1 + assert fallback_records == collector_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("direct", [False, True]) +def test_in_opentelemetry_logs_direct_cfl_threaded_stress(tmp_path, direct): + request_count = 200 + processors = DIRECT_CFL_PROCESSOR_CHAINS["hash_once"] + service = direct_cfl_service( + tmp_path, direct, processors, workers=4, threaded=True, + conditional_routes=True) + payload = build_direct_cfl_sentinel_payload().SerializeToString() + + try: + service.start() + before = len(data_storage["logs"]) + + def send_request(_): + return service.send_raw_request("/v1/logs", payload).status_code + + with ThreadPoolExecutor(max_workers=16) as executor: + statuses = list(executor.map(send_request, range(request_count))) + + assert statuses == [201] * request_count + + def received_records(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if len(records) >= request_count else None + + records = service.service.wait_for_condition( + received_records, timeout=30, interval=0.1, + description="threaded direct-CFL stress records") + assert len(records) == request_count + finally: + service.stop() + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("chain", ["hash_then_filter", "filter_first", + "hash_then_unflagged", "unflagged_then_hash"]) +def test_in_opentelemetry_logs_direct_cfl_workers_fallback(tmp_path, chain): + # Hashing is intentionally non-idempotent. If an eligible prefix ran in a + # worker before the whole chain fell back, direct-on parity would fail. + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + msgpack_records = run_direct_cfl_case(tmp_path, False, processors, "http", workers=4) + direct_records = run_direct_cfl_case(tmp_path, True, processors, "http", workers=4) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("chain", ["content_modifier", "sql_drop", "hash_then_sql", + "conditional_safe"]) +def test_in_opentelemetry_logs_direct_cfl_threaded_worker_safe_chain(tmp_path, chain): + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + msgpack_records = run_direct_cfl_case( + tmp_path, False, processors, "http", workers=4, threaded=True) + direct_records = run_direct_cfl_case( + tmp_path, True, processors, "http", workers=4, threaded=True) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.parametrize("direct", [False, True]) +def test_in_opentelemetry_logs_direct_cfl_drop_all(tmp_path, direct): + # Every record of the payload is dropped, only the sentinel matches. + processors = [{"name": "sql", "query": "SELECT * FROM STREAM WHERE keep = 'last';"}] + records = run_direct_cfl_case(tmp_path, direct, processors, "http", sentinel_keep="last") + + assert len(records) == 1 + assert is_sentinel(records[0]) + + +@pytest.mark.parametrize("attributes,scopes,records,resources,accepted", [ + (1, 32, 1, 1, True), + (64, 1, 1, 1, True), + (64, 32, 1, 1, False), + (1, 1, 512, 1, False), + # scopes without records produce nothing, the request is accepted + (64, 1000, 0, 1, True), +]) +def test_in_opentelemetry_logs_direct_cfl_generated_limit(tmp_path, attributes, scopes, + records, resources, accepted): + processors = DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"] + service = direct_cfl_service(tmp_path, True, processors, logs_max_size="16k") + request = generated_limit_payload(attributes, scopes, records, resources) + try: + service.start() + response = service.send_raw_request("/v1/logs", request.SerializeToString()) + assert response.status_code == (201 if accepted else 400) + + # A rejected request never appends a prefix of its records. + small = generated_limit_payload(1, 1) + small.resource_logs[0].scope_logs[0].log_records[0].body.string_value = "recovery" + response = service.send_raw_request("/v1/logs", small.SerializeToString()) + assert response.status_code == 201 + + def received_records(): + return [record for request in data_storage["logs"] + for resource in request.resource_logs + for scope in resource.scope_logs for record in scope.log_records] + + expected = scopes * records * resources if accepted else 0 + service.service.wait_for_condition( + lambda: len(received_records()) >= expected + 1, + timeout=20, interval=0.1, description="recovery after direct CFL byte limit") + actual = received_records() + assert sum(record.body.string_value == "recovery" for record in actual) == 1 + assert sum(record.body.string_value == "generated-limit" for record in actual) == expected + finally: + service.stop() + + +@pytest.mark.parametrize("limit", [512, 1024, 2048, 4096, 8192, 16384]) +def test_in_opentelemetry_logs_direct_cfl_limit_parity(tmp_path, limit): + processors = DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"] + payload = build_direct_cfl_logs_payload().SerializeToString() + statuses = [] + + for direct in [False, True]: + service = direct_cfl_service(tmp_path, direct, processors, + logs_max_size=str(limit)) + try: + service.start() + response = service.send_raw_request("/v1/logs", payload) + statuses.append(response.status_code) + finally: + service.stop() + + assert statuses[1] == statuses[0] diff --git a/tests/integration/scenarios/out_stdout/config/out_stdout_processor_workers.yaml b/tests/integration/scenarios/out_stdout/config/out_stdout_processor_workers.yaml new file mode 100644 index 00000000000..1f055be9ae0 --- /dev/null +++ b/tests/integration/scenarios/out_stdout/config/out_stdout_processor_workers.yaml @@ -0,0 +1,48 @@ +service: + flush: 0.1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: output_single + dummy: '{"message":"output worker single"}' + samples: 1 + + - name: dummy + tag: output_chain + dummy: '{"message":"output worker chain"}' + samples: 1 + + outputs: + - name: stdout + match: output_single + workers: 4 + processors: + logs: + - name: content_modifier + context: body + action: insert + key: native_one + value: yes + + - name: stdout + match: output_chain + workers: 4 + processors: + logs: + - name: content_modifier + context: body + action: insert + key: native_one + value: yes + - name: modify + add: filter_middle yes + - name: content_modifier + context: body + action: insert + key: native_two + value: yes diff --git a/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py b/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py index 33d847019a7..81262c11570 100644 --- a/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py +++ b/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py @@ -201,6 +201,27 @@ def test_out_stdout_logs_accepts_otlp_json_ingestion(): assert "This is another example log message." in log_text +def test_out_stdout_processor_chains_with_four_output_workers(): + service = Service("out_stdout_processor_workers.yaml") + service.start() + service.wait_for_log_contains("output worker single") + service.wait_for_log_contains("output worker chain") + log_text = service.read_log() + service.stop() + + # Concurrent stdout workers can interleave writes at byte granularity, so + # validate the complete captured stream rather than individual lines. + assert "] output_single:" in log_text + assert "] output_chain:" in log_text + assert log_text.count('"native_one"=>"yes"') == 2 + assert '"filter_middle"=>"yes"' in log_text + assert log_text.count('"native_two"=>"yes"') == 1 + for output_index in (0, 1): + for worker_index in range(4): + assert (f"[output:stdout:stdout.{output_index}] worker " + f"#{worker_index} started") in log_text + + def test_out_stdout_preserves_post_2038_otlp_timestamp(): service = Service("out_stdout_otel.yaml") service.start() From a3b11962671c223581dad2020c561ca3183cb55c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 27 Oct 2026 22:19:31 -0600 Subject: [PATCH 12/33] opentelemetry: reuse protobuf arena for direct logs Signed-off-by: Eduardo Silva --- .../flb_opentelemetry_logs_proto.c | 66 ++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/opentelemetry/flb_opentelemetry_logs_proto.c b/src/opentelemetry/flb_opentelemetry_logs_proto.c index 3cc9bd99e0f..e91050fa824 100644 --- a/src/opentelemetry/flb_opentelemetry_logs_proto.c +++ b/src/opentelemetry/flb_opentelemetry_logs_proto.c @@ -99,6 +99,14 @@ static void protobuf_logs_free( Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *logs, ProtobufCAllocator *allocator) { + /* + * Every allocation of an arena backed request is released with the arena, + * walking the message tree here would only call the no-op free callback. + */ + if (allocator != NULL && allocator->free == protobuf_arena_free) { + return; + } + opentelemetry__proto__collector__logs__v1__export_logs_service_request__free_unpacked( logs, allocator); } @@ -826,6 +834,7 @@ int flb_opentelemetry_logs_protobuf_to_msgpack( struct otel_cfl_builder { const struct flb_opentelemetry_logs_decode_options *options; struct flb_mp_chunk_cobj *chunk; + struct cfl_arena *arena; size_t size; size_t limit; }; @@ -933,7 +942,7 @@ static struct cfl_variant *otel_cfl_string(struct otel_cfl_builder *builder, return NULL; } - return cfl_variant_create_from_string_s(str, len, CFL_TRUE); + return cfl_variant_create_from_string_s_in(builder->arena, str, len, CFL_TRUE); } static struct cfl_variant *otel_cfl_bytes(struct otel_cfl_builder *builder, @@ -945,7 +954,8 @@ static struct cfl_variant *otel_cfl_bytes(struct otel_cfl_builder *builder, return NULL; } - return cfl_variant_create_from_bytes((char *) bytes.data, bytes.len, CFL_TRUE); + return cfl_variant_create_from_bytes_in(builder->arena, + (char *) bytes.data, bytes.len, CFL_TRUE); } static struct cfl_variant *otel_cfl_uint(struct otel_cfl_builder *builder, uint64_t val) @@ -954,7 +964,7 @@ static struct cfl_variant *otel_cfl_uint(struct otel_cfl_builder *builder, uint6 return NULL; } - return cfl_variant_create_from_uint64(val); + return cfl_variant_create_from_uint64_in(builder->arena, val); } /* msgpack keeps non-negative integers as unsigned values */ @@ -968,7 +978,7 @@ static struct cfl_variant *otel_cfl_int(struct otel_cfl_builder *builder, int64_ return NULL; } - return cfl_variant_create_from_int64(val); + return cfl_variant_create_from_int64_in(builder->arena, val); } /* Insert a value, the value is released when it cannot be inserted */ @@ -1002,7 +1012,7 @@ static int otel_cfl_insert_kvlist(struct otel_cfl_builder *builder, return -1; } - variant = cfl_variant_create_from_kvlist(value); + variant = cfl_variant_create_from_kvlist_in(builder->arena, value); if (variant == NULL) { cfl_kvlist_destroy(value); return -1; @@ -1018,7 +1028,7 @@ static struct cfl_kvlist *otel_cfl_kvlist_create(struct otel_cfl_builder *builde return NULL; } - return cfl_kvlist_create(); + return cfl_kvlist_create_in(builder->arena); } /* A map the msgpack path packs with a compact header */ @@ -1029,7 +1039,7 @@ static struct cfl_kvlist *otel_cfl_kvlist_create_compact(struct otel_cfl_builder return NULL; } - return cfl_kvlist_create(); + return cfl_kvlist_create_in(builder->arena); } static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, @@ -1072,7 +1082,7 @@ static struct cfl_variant *otel_cfl_array(struct otel_cfl_builder *builder, return NULL; } - entries = cfl_array_create(array->n_values); + entries = cfl_array_create_in(builder->arena, array->n_values); if (entries == NULL) { return NULL; } @@ -1088,7 +1098,7 @@ static struct cfl_variant *otel_cfl_array(struct otel_cfl_builder *builder, } } - variant = cfl_variant_create_from_array(entries); + variant = cfl_variant_create_from_array_in(builder->arena, entries); if (variant == NULL) { cfl_array_destroy(entries); } @@ -1106,7 +1116,7 @@ static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, if (otel_cfl_account(builder, 1) != 0) { return NULL; } - return cfl_variant_create_from_null(); + return cfl_variant_create_from_null_in(builder->arena); } switch (value->value_case) { @@ -1117,7 +1127,7 @@ static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, if (otel_cfl_account(builder, 1) != 0) { return NULL; } - return cfl_variant_create_from_bool(value->bool_value); + return cfl_variant_create_from_bool_in(builder->arena, value->bool_value); case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: return otel_cfl_int(builder, value->int_value); @@ -1126,7 +1136,7 @@ static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, if (otel_cfl_account(builder, 9) != 0) { return NULL; } - return cfl_variant_create_from_double(value->double_value); + return cfl_variant_create_from_double_in(builder->arena, value->double_value); case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: return otel_cfl_array(builder, value->array_value); @@ -1138,7 +1148,7 @@ static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, return NULL; } - variant = cfl_variant_create_from_kvlist(kvlist); + variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); if (variant == NULL) { cfl_kvlist_destroy(kvlist); } @@ -1153,7 +1163,7 @@ static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, if (otel_cfl_account(builder, 1) != 0) { return NULL; } - return cfl_variant_create_from_null(); + return cfl_variant_create_from_null_in(builder->arena); default: flb_error("[otel]: invalid value type in pack_any_value"); @@ -1162,7 +1172,8 @@ static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, } /* Wrap a kvlist as a record object, the kvlist is released on failure */ -static struct cfl_object *otel_cfl_object(struct cfl_kvlist *kvlist) +static struct cfl_object *otel_cfl_object(struct otel_cfl_builder *builder, + struct cfl_kvlist *kvlist) { struct cfl_object *object; @@ -1176,7 +1187,7 @@ static struct cfl_object *otel_cfl_object(struct cfl_kvlist *kvlist) return NULL; } - object->variant = cfl_variant_create_from_kvlist(kvlist); + object->variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); if (object->variant == NULL) { cfl_kvlist_destroy(kvlist); cfl_object_destroy(object); @@ -1475,9 +1486,10 @@ static int otel_cfl_scope_logs(struct otel_cfl_builder *builder, return -1; } - group->cobj_metadata = otel_cfl_object( + group->cobj_metadata = otel_cfl_object(builder, otel_cfl_group_metadata(builder, resource_index, scope_index)); - group->cobj_record = otel_cfl_object(otel_cfl_group_body(builder, resource_log, scope_log)); + group->cobj_record = otel_cfl_object(builder, + otel_cfl_group_body(builder, resource_log, scope_log)); if (group->cobj_metadata == NULL || group->cobj_record == NULL) { return -1; } @@ -1510,13 +1522,15 @@ static int otel_cfl_scope_logs(struct otel_cfl_builder *builder, return -1; } - record->cobj_metadata = otel_cfl_object(otel_cfl_v1_metadata(builder, log_record)); + record->cobj_metadata = otel_cfl_object(builder, + otel_cfl_v1_metadata(builder, log_record)); if (record->cobj_metadata == NULL) { flb_error("[opentelemetry] failed to convert log record"); return -1; } - record->cobj_record = otel_cfl_object(otel_cfl_body(builder, log_record->body)); + record->cobj_record = otel_cfl_object(builder, + otel_cfl_body(builder, log_record->body)); if (record->cobj_record == NULL) { flb_error("[opentelemetry] failed to convert log record body"); return -1; @@ -1533,8 +1547,8 @@ static int otel_cfl_scope_logs(struct otel_cfl_builder *builder, return -1; } - record->cobj_metadata = otel_cfl_object(otel_cfl_kvlist_create(builder)); - record->cobj_record = otel_cfl_object(otel_cfl_kvlist_create(builder)); + record->cobj_metadata = otel_cfl_object(builder, otel_cfl_kvlist_create(builder)); + record->cobj_record = otel_cfl_object(builder, otel_cfl_kvlist_create(builder)); if (record->cobj_metadata == NULL || record->cobj_record == NULL) { return -1; } @@ -1585,6 +1599,7 @@ int flb_opentelemetry_logs_protobuf_to_cfl( } builder.options = options; + builder.arena = NULL; builder.size = 0; builder.limit = options->output_limit; builder.chunk = NULL; @@ -1600,6 +1615,13 @@ int flb_opentelemetry_logs_protobuf_to_cfl( ret = -1; goto end; } + /* + * The CFL tree is consumed and encoded synchronously before the decoded + * protobuf request is released, so both representations can safely share + * the request arena. This avoids one heap allocation per CFL container and + * scalar while preserving the normal allocator fallback for small payloads. + */ + builder.arena = result->protobuf_arena; if (result->input_logs->n_resource_logs > 0 && result->input_logs->resource_logs == NULL) { From 12a1ace38c890dfa0369c634dbd1f683914df1de Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 06:19:42 -0600 Subject: [PATCH 13/33] processor: restrict direct CFL to native chains Signed-off-by: Eduardo Silva --- src/flb_processor.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/flb_processor.c b/src/flb_processor.c index 59a4a89ea04..9e9cfaa55e4 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -2535,10 +2535,11 @@ int flb_processor_run_with_count(struct flb_processor *proc, /* * A logs chunk built directly as CFL records can enter the pipeline when the - * first unit is a native processor and every native unit processes logs. A - * worker thread additionally requires every unit to be a worker-safe native - * processor. The whole chain is checked before any unit runs so fallback can - * never execute an already-processed prefix a second time. + * every unit is a native processor that processes logs. A worker thread also + * requires every unit to be worker-safe. The whole chain is checked before + * any unit runs so fallback can never execute an already-processed prefix a + * second time. Filters are excluded even in non-worker callers because mixed + * native/filter chains cross a serialization boundary between units. */ int flb_processor_logs_chunk_supported(struct flb_processor *proc, int worker_thread) @@ -2554,10 +2555,7 @@ int flb_processor_logs_chunk_supported(struct flb_processor *proc, mk_list_foreach(head, &proc->logs) { pu = mk_list_entry(head, struct flb_processor_unit, _head); if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { - if (head == proc->logs.next || worker_thread == FLB_TRUE) { - return FLB_FALSE; - } - continue; + return FLB_FALSE; } p_ins = pu->ctx; From 396f3149eccb8e83a773e8dc1964b9a3a7cd1657 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 06:19:42 -0600 Subject: [PATCH 14/33] processor_content_modifier: preserve OTLP container arenas Signed-off-by: Eduardo Silva --- plugins/processor_content_modifier/cm_opentelemetry.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/processor_content_modifier/cm_opentelemetry.c b/plugins/processor_content_modifier/cm_opentelemetry.c index 3cb3b60bd80..792793ea114 100644 --- a/plugins/processor_content_modifier/cm_opentelemetry.c +++ b/plugins/processor_content_modifier/cm_opentelemetry.c @@ -59,7 +59,7 @@ struct cfl_variant *cm_otel_get_or_create_attributes(struct cfl_kvlist *kvlist) } /* create an empty kvlist as the value of attributes */ - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } @@ -106,7 +106,7 @@ static struct cfl_variant *otel_get_or_create_attributes(struct cfl_kvlist *kvli } /* create an empty kvlist as the value of attributes */ - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } @@ -188,7 +188,7 @@ static struct cfl_variant *otel_get_or_create_scope_metadata(int telemetry_type, } /* metadata don't exists, create it */ - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } @@ -237,7 +237,7 @@ struct cfl_variant *cm_otel_get_scope_metadata(int telemetry_type, struct cfl_kv } else { /* create "scope" inside kvlist */ - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } From 2923d12322b6ab237e1b318401ce3d2a43f83081 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 06:19:42 -0600 Subject: [PATCH 15/33] processor_content_modifier: preserve log value arenas Signed-off-by: Eduardo Silva --- plugins/processor_content_modifier/cm_logs.c | 45 +++++++++++++++++++- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/plugins/processor_content_modifier/cm_logs.c b/plugins/processor_content_modifier/cm_logs.c index 83029e3d71e..3c2a4c1ac61 100644 --- a/plugins/processor_content_modifier/cm_logs.c +++ b/plugins/processor_content_modifier/cm_logs.c @@ -253,6 +253,7 @@ static int run_action_convert(struct content_modifier_ctx *ctx, struct cfl_kvpair *kvpair; struct cfl_variant *v; struct cfl_variant *converted; + struct cfl_variant *arena_converted; /* if the kv pair already exists, remove it from the list */ kvpair = cfl_object_kvpair_get(obj, key); @@ -267,10 +268,50 @@ static int run_action_convert(struct content_modifier_ctx *ctx, return -1; } + kvlist = obj->variant->data.as_kvlist; + if (kvlist->arena != NULL) { + arena_converted = NULL; + + if (converted->type == CFL_VARIANT_STRING) { + arena_converted = cfl_variant_create_from_string_s_in( + kvlist->arena, converted->data.as_string, + cfl_variant_size_get(converted), CFL_FALSE); + } + else if (converted->type == CFL_VARIANT_BYTES) { + arena_converted = cfl_variant_create_from_bytes_in( + kvlist->arena, converted->data.as_bytes, + cfl_variant_size_get(converted), CFL_FALSE); + } + else if (converted->type == CFL_VARIANT_BOOL) { + arena_converted = cfl_variant_create_from_bool_in( + kvlist->arena, converted->data.as_bool); + } + else if (converted->type == CFL_VARIANT_INT) { + arena_converted = cfl_variant_create_from_int64_in( + kvlist->arena, converted->data.as_int64); + } + else if (converted->type == CFL_VARIANT_UINT) { + arena_converted = cfl_variant_create_from_uint64_in( + kvlist->arena, converted->data.as_uint64); + } + else if (converted->type == CFL_VARIANT_DOUBLE) { + arena_converted = cfl_variant_create_from_double_in( + kvlist->arena, converted->data.as_double); + } + else if (converted->type == CFL_VARIANT_NULL) { + arena_converted = cfl_variant_create_from_null_in(kvlist->arena); + } + + cfl_variant_destroy(converted); + if (arena_converted == NULL) { + return -1; + } + converted = arena_converted; + } + /* remove the old kvpair */ cfl_kvpair_destroy(kvpair); - kvlist = obj->variant->data.as_kvlist; ret = cfl_kvlist_insert_s(kvlist, key, cfl_sds_len(key), converted); if (ret != 0) { cfl_variant_destroy(converted); @@ -326,7 +367,7 @@ static struct cfl_variant *otel_get_log_attributes(struct flb_mp_chunk_record *r } } else { - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } From bcbdabb5c12c07d0efdc635e33834e8953d16666 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 06:19:42 -0600 Subject: [PATCH 16/33] opentelemetry: decode JSON logs directly to CFL Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_opentelemetry.h | 23 + src/CMakeLists.txt | 1 + .../flb_opentelemetry_logs_json.c | 1511 +++++++++++++++++ 3 files changed, 1535 insertions(+) create mode 100644 src/opentelemetry/flb_opentelemetry_logs_json.c diff --git a/include/fluent-bit/flb_opentelemetry.h b/include/fluent-bit/flb_opentelemetry.h index f36e55d1415..ff214970839 100644 --- a/include/fluent-bit/flb_opentelemetry.h +++ b/include/fluent-bit/flb_opentelemetry.h @@ -129,6 +129,7 @@ struct flb_opentelemetry_logs_decode_options { struct flb_mp_chunk_cobj; struct flb_opentelemetry_logs_protobuf_cfl; +struct flb_opentelemetry_logs_json_cfl; /* Backward-compatible alias for older external callers. */ #define flb_opentelemetry_otlp_json_options flb_opentelemetry_otlp_logs_options @@ -215,6 +216,28 @@ int flb_opentelemetry_logs_json_to_msgpack(struct flb_log_event_encoder *encoder const char *logs_body_key, int *error_status); +/* + * Returned by flb_opentelemetry_logs_json_to_cfl() for a valid payload whose + * records CFL cannot hold, such as a map key that is not a string. Use + * flb_opentelemetry_logs_json_to_msgpack() for it. + */ +#define FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED -2 + +int flb_opentelemetry_logs_json_to_cfl( + const char *body, size_t len, + const struct flb_opentelemetry_logs_decode_options *options, + struct flb_opentelemetry_logs_json_cfl **result, + int *error_status); + +struct flb_mp_chunk_cobj *flb_opentelemetry_logs_json_cfl_take_chunk( + struct flb_opentelemetry_logs_json_cfl *result); + +size_t flb_opentelemetry_logs_json_cfl_get_encoded_size( + struct flb_opentelemetry_logs_json_cfl *result); + +void flb_opentelemetry_logs_json_cfl_destroy( + struct flb_opentelemetry_logs_json_cfl *result); + int flb_opentelemetry_protobuf_validate( const ProtobufCMessageDescriptor *descriptor, const void *data, size_t size); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 819a6848204..324ae6088a5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -128,6 +128,7 @@ endif() set(src ${src} opentelemetry/flb_opentelemetry_logs.c + opentelemetry/flb_opentelemetry_logs_json.c opentelemetry/flb_opentelemetry_logs_proto.c opentelemetry/flb_opentelemetry_metrics.c opentelemetry/flb_opentelemetry_protobuf.c diff --git a/src/opentelemetry/flb_opentelemetry_logs_json.c b/src/opentelemetry/flb_opentelemetry_logs_json.c new file mode 100644 index 00000000000..04e83567f80 --- /dev/null +++ b/src/opentelemetry/flb_opentelemetry_logs_json.c @@ -0,0 +1,1511 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#define OTEL_JSON_CFL_ARENA_CHUNK_SIZE 16384 +/* + * The established path builds maps and arrays with 32-bit headers that are + * patched once the entries are known, the logs_max_size accounting counts + * them the same way. Values it copies unchanged keep compact headers. + */ +#define OTEL_JSON_CFL_MAP_HEADER_SIZE 5 +#define OTEL_JSON_CFL_ARRAY_HEADER_SIZE 5 +#define OTEL_JSON_CFL_RECORD_SIZE 12 +#define OTEL_JSON_CFL_MAX_DEPTH 32 + +struct json_cfl_builder { + const struct flb_opentelemetry_logs_decode_options *options; + struct flb_mp_chunk_cobj *chunk; + struct cfl_arena *arena; + size_t size; + size_t limit; + int error_status; + int unsupported; +}; + +struct flb_opentelemetry_logs_json_cfl { + struct flb_mp_chunk_cobj *chunk; + struct cfl_arena *arena; + yyjson_doc *document; + size_t encoded_size; + size_t record_count; +}; + +static void *json_arena_malloc(void *context, size_t size) +{ + (void) context; + return flb_malloc(size); +} + +static void json_arena_free(void *context, void *pointer) +{ + (void) context; + flb_free(pointer); +} + +static struct cfl_arena *json_arena_create(void) +{ + struct cfl_arena_options options; + + cfl_arena_options_init(&options); + options.chunk_size = OTEL_JSON_CFL_ARENA_CHUNK_SIZE; + options.maximum_chunk_size = 65536; + options.malloc_fn = json_arena_malloc; + options.free_fn = json_arena_free; + + return cfl_arena_create_with_options(&options); +} + +static int json_cfl_account(struct json_cfl_builder *builder, size_t bytes) +{ + if (bytes > builder->limit || builder->size > builder->limit - bytes) { + return -1; + } + builder->size += bytes; + return 0; +} + +static size_t json_cfl_uint_size(uint64_t value) +{ + if (value <= 0x7f) { + return 1; + } + if (value <= UINT8_MAX) { + return 2; + } + if (value <= UINT16_MAX) { + return 3; + } + if (value <= UINT32_MAX) { + return 5; + } + return 9; +} + +static size_t json_cfl_int_size(int64_t value) +{ + if (value >= -32) { + return 1; + } + if (value >= INT8_MIN) { + return 2; + } + if (value >= INT16_MIN) { + return 3; + } + if (value >= INT32_MIN) { + return 5; + } + return 9; +} + +static size_t json_cfl_string_header_size(size_t length) +{ + if (length <= 31) { + return 1; + } + if (length <= UINT8_MAX) { + return 2; + } + if (length <= UINT16_MAX) { + return 3; + } + return 5; +} + +static size_t json_cfl_binary_header_size(size_t length) +{ + if (length <= UINT8_MAX) { + return 2; + } + if (length <= UINT16_MAX) { + return 3; + } + return 5; +} + +static size_t json_cfl_array_header_size(size_t entries) +{ + if (entries <= 15) { + return 1; + } + if (entries <= UINT16_MAX) { + return 3; + } + return 5; +} + +static int json_cfl_account_sized(struct json_cfl_builder *builder, + size_t header, size_t value) +{ + if (json_cfl_account(builder, header) != 0) { + return -1; + } + return json_cfl_account(builder, value); +} + +static struct cfl_variant *json_cfl_string(struct json_cfl_builder *builder, + const char *value, size_t length) +{ + if (json_cfl_account_sized(builder, json_cfl_string_header_size(length), length) != 0) { + return NULL; + } + return cfl_variant_create_from_string_s_in(builder->arena, + (char *) value, length, CFL_TRUE); +} + +static struct cfl_variant *json_cfl_uint(struct json_cfl_builder *builder, + uint64_t value) +{ + if (json_cfl_account(builder, json_cfl_uint_size(value)) != 0) { + return NULL; + } + return cfl_variant_create_from_uint64_in(builder->arena, value); +} + +static struct cfl_variant *json_cfl_int(struct json_cfl_builder *builder, + int64_t value) +{ + if (value >= 0) { + return json_cfl_uint(builder, (uint64_t) value); + } + if (json_cfl_account(builder, json_cfl_int_size(value)) != 0) { + return NULL; + } + return cfl_variant_create_from_int64_in(builder->arena, value); +} + +static struct cfl_variant *json_cfl_double(struct json_cfl_builder *builder, + double value) +{ + if (json_cfl_account(builder, 9) != 0) { + return NULL; + } + return cfl_variant_create_from_double_in(builder->arena, value); +} + +static struct cfl_variant *json_cfl_bool(struct json_cfl_builder *builder, int value) +{ + if (json_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_bool_in(builder->arena, value); +} + +static struct cfl_variant *json_cfl_null(struct json_cfl_builder *builder) +{ + if (json_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_null_in(builder->arena); +} + +static struct cfl_kvlist *json_cfl_kvlist_create(struct json_cfl_builder *builder) +{ + if (json_cfl_account(builder, OTEL_JSON_CFL_MAP_HEADER_SIZE) != 0) { + return NULL; + } + return cfl_kvlist_create_in(builder->arena); +} + +static int json_cfl_insert(struct json_cfl_builder *builder, + struct cfl_kvlist *kvlist, + const char *key, size_t key_length, + struct cfl_variant *value) +{ + if (value == NULL) { + return -1; + } + if (json_cfl_account_sized(builder, json_cfl_string_header_size(key_length), + key_length) != 0 || + cfl_kvlist_insert_s(kvlist, (char *) key, key_length, value) != 0) { + cfl_variant_destroy(value); + return -1; + } + return 0; +} + +static int json_cfl_insert_kvlist(struct json_cfl_builder *builder, + struct cfl_kvlist *target, + const char *key, size_t key_length, + struct cfl_kvlist *value) +{ + struct cfl_variant *variant; + + if (value == NULL) { + return -1; + } + variant = cfl_variant_create_from_kvlist_in(builder->arena, value); + if (variant == NULL) { + cfl_kvlist_destroy(value); + return -1; + } + return json_cfl_insert(builder, target, key, key_length, variant); +} + +static struct cfl_object *json_cfl_object(struct json_cfl_builder *builder, + struct cfl_kvlist *kvlist) +{ + struct cfl_object *object; + + if (kvlist == NULL) { + return NULL; + } + object = cfl_object_create(); + if (object == NULL) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + object->variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); + if (object->variant == NULL) { + cfl_kvlist_destroy(kvlist); + cfl_object_destroy(object); + return NULL; + } + object->type = CFL_OBJECT_KVLIST; + return object; +} + +static struct flb_mp_chunk_record *json_cfl_record_add(struct json_cfl_builder *builder, + struct flb_time *timestamp) +{ + struct flb_mp_chunk_record *record; + + if (json_cfl_account(builder, OTEL_JSON_CFL_RECORD_SIZE) != 0) { + return NULL; + } + record = flb_mp_chunk_record_create(builder->chunk); + if (record == NULL) { + return NULL; + } + memset(&record->event, 0, sizeof(record->event)); + flb_time_copy(&record->event.timestamp, timestamp); + cfl_list_add(&record->_head, &builder->chunk->records); + return record; +} + +/* + * Keys are compared case-insensitively. The expected length is known at + * compile time, so most keys are rejected by their length alone; callers + * pass string literals through the json_key_equal() and json_obj_get() + * macros. + */ +static int json_key_equal_n(yyjson_val *key, const char *expected, size_t expected_length) +{ + if (!yyjson_is_str(key) || yyjson_get_len(key) != expected_length) { + return FLB_FALSE; + } + return strncasecmp(yyjson_get_str(key), expected, expected_length) == 0; +} + +static yyjson_val *json_obj_get_n(yyjson_val *object, const char *name, size_t name_length) +{ + size_t index; + size_t maximum; + yyjson_val *key; + yyjson_val *value; + + if (!yyjson_is_obj(object)) { + return NULL; + } + yyjson_obj_foreach(object, index, maximum, key, value) { + if (json_key_equal_n(key, name, name_length)) { + return value; + } + } + return NULL; +} + +#define json_key_equal(key, literal) json_key_equal_n(key, "" literal, sizeof(literal) - 1) +#define json_obj_get(object, literal) json_obj_get_n(object, "" literal, sizeof(literal) - 1) + +static int json_base64_decode(const char *source, size_t source_size, + unsigned char **output, size_t *output_size) +{ + int result; + char value; + char *normalized; + size_t index; + size_t padded_size; + unsigned char *decoded; + + *output = NULL; + *output_size = 0; + if (source_size % 4 == 1) { + return -1; + } + padded_size = source_size + ((4 - source_size % 4) % 4); + normalized = flb_malloc(padded_size + 1); + decoded = flb_malloc((padded_size / 4) * 3 + 1); + if (normalized == NULL || decoded == NULL) { + flb_free(normalized); + flb_free(decoded); + return -1; + } + for (index = 0; index < source_size; index++) { + value = source[index]; + normalized[index] = value == '-' ? '+' : (value == '_' ? '/' : value); + } + for (; index < padded_size; index++) { + normalized[index] = '='; + } + normalized[padded_size] = '\0'; + result = flb_base64_decode(decoded, (padded_size / 4) * 3 + 1, output_size, + (unsigned char *) normalized, padded_size); + flb_free(normalized); + if (result != 0) { + flb_free(decoded); + return -1; + } + *output = decoded; + return 0; +} + +static struct cfl_variant *json_plain_value(struct json_cfl_builder *builder, + yyjson_val *value); +static struct cfl_variant *json_converted_value(struct json_cfl_builder *builder, + yyjson_val *value, size_t depth); + +/* + * The conversion of OTLP JSON values below mirrors the established msgpack + * path in flb_opentelemetry_utils.c function by function, including its + * leniency, so both paths produce the same records: + * + * json_wrapped() flb_otel_utils_json_payload_get_wrapped_value() + * json_unwrapped_value() flb_otel_utils_json_payload_append_unwrapped_value() + * json_converted_*() flb_otel_utils_json_payload_append_converted_*() + * + * A result CFL cannot hold, a map key that is not a string, marks the + * payload unsupported so the caller can use the msgpack path instead. + */ +#define JSON_WRAPPED_STRING 1 +#define JSON_WRAPPED_BOOL 2 +#define JSON_WRAPPED_INT 3 +#define JSON_WRAPPED_DOUBLE 4 +#define JSON_WRAPPED_BYTES 5 +#define JSON_WRAPPED_ARRAY 6 +#define JSON_WRAPPED_KVLIST 7 + +/* like the established path, a wrapper key matches any prefix of the name */ +static int json_wrapper_key(yyjson_val *key, const char *name) +{ + return strncasecmp(yyjson_get_str(key), name, yyjson_get_len(key)) == 0; +} + +static int json_wrapped(yyjson_val *object, yyjson_val **out_value, int *out_type) +{ + int type; + yyjson_val *key; + yyjson_val *value; + yyjson_obj_iter iterator; + + if (!yyjson_is_obj(object)) { + return -1; + } + type = -1; + value = NULL; + if (yyjson_obj_size(object) == 1) { + yyjson_obj_iter_init(object, &iterator); + key = yyjson_obj_iter_next(&iterator); + value = yyjson_obj_iter_get_val(key); + + if (json_wrapper_key(key, "stringValue")) { + if (!yyjson_is_null(value) && !yyjson_is_str(value)) { + return -2; + } + type = JSON_WRAPPED_STRING; + } + else if (json_wrapper_key(key, "boolValue")) { + if (!yyjson_is_bool(value)) { + return -2; + } + type = JSON_WRAPPED_BOOL; + } + else if (json_wrapper_key(key, "intValue")) { + if (!yyjson_is_int(value) && !yyjson_is_str(value)) { + return -2; + } + type = JSON_WRAPPED_INT; + } + else if (json_wrapper_key(key, "doubleValue")) { + if (!yyjson_is_real(value) && !yyjson_is_str(value)) { + return -2; + } + type = JSON_WRAPPED_DOUBLE; + } + else if (json_wrapper_key(key, "bytesValue")) { + if (!yyjson_is_str(value)) { + return -2; + } + type = JSON_WRAPPED_BYTES; + } + else if (json_wrapper_key(key, "arrayValue")) { + if (!yyjson_is_arr(value) && !yyjson_is_obj(value)) { + return -2; + } + type = JSON_WRAPPED_ARRAY; + } + else if (json_wrapper_key(key, "kvlistValue")) { + if (!yyjson_is_obj(value)) { + return -2; + } + type = JSON_WRAPPED_KVLIST; + } + } + if (type == -1) { + return -1; + } + if (out_type != NULL) { + *out_type = type; + } + if (yyjson_is_obj(value) && yyjson_obj_size(value) == 1) { + yyjson_obj_iter_init(value, &iterator); + key = yyjson_obj_iter_next(&iterator); + if (yyjson_get_len(key) != 6 || strncasecmp(yyjson_get_str(key), "values", 6) != 0) { + return -3; + } + value = yyjson_obj_iter_get_val(key); + } + if (type == JSON_WRAPPED_ARRAY && !yyjson_is_arr(value) && + (!yyjson_is_obj(value) || yyjson_obj_size(value) != 0)) { + return -2; + } + else if (type == JSON_WRAPPED_KVLIST && !yyjson_is_arr(value) && !yyjson_is_obj(value)) { + return -2; + } + if (out_value != NULL) { + *out_value = value; + } + return 0; +} + +static struct cfl_variant *json_converted_array(struct json_cfl_builder *builder, + yyjson_val *array, size_t depth) +{ + size_t index; + size_t maximum; + yyjson_val *entry; + struct cfl_array *result; + struct cfl_variant *variant; + + if (depth > OTEL_JSON_CFL_MAX_DEPTH) { + builder->unsupported = FLB_TRUE; + return NULL; + } + if (json_cfl_account(builder, OTEL_JSON_CFL_ARRAY_HEADER_SIZE) != 0) { + return NULL; + } + result = cfl_array_create_in(builder->arena, yyjson_arr_size(array)); + if (result == NULL) { + return NULL; + } + yyjson_arr_foreach(array, index, maximum, entry) { + variant = json_converted_value(builder, entry, depth + 1); + if (variant == NULL || cfl_array_append(result, variant) != 0) { + if (variant != NULL) { + cfl_variant_destroy(variant); + } + cfl_array_destroy(result); + return NULL; + } + } + variant = cfl_variant_create_from_array_in(builder->arena, result); + if (variant == NULL) { + cfl_array_destroy(result); + } + return variant; +} + +static struct cfl_variant *json_kvlist_variant(struct json_cfl_builder *builder, + struct cfl_kvlist *kvlist) +{ + struct cfl_variant *variant; + + if (kvlist == NULL) { + return NULL; + } + variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); + if (variant == NULL) { + cfl_kvlist_destroy(kvlist); + } + return variant; +} + +/* Insert a converted key, CFL only holds string keys */ +static int json_converted_insert(struct json_cfl_builder *builder, struct cfl_kvlist *kvlist, + yyjson_val *key, struct cfl_variant *value, size_t depth) +{ + int result; + size_t size; + struct cfl_variant *key_variant; + + if (value == NULL) { + return -1; + } + if (yyjson_is_str(key)) { + return json_cfl_insert(builder, kvlist, yyjson_get_str(key), yyjson_get_len(key), + value); + } + + /* the key is accounted once, by the insertion */ + size = builder->size; + key_variant = json_converted_value(builder, key, depth + 1); + builder->size = size; + if (key_variant == NULL || key_variant->type != CFL_VARIANT_STRING) { + if (key_variant != NULL) { + builder->unsupported = FLB_TRUE; + cfl_variant_destroy(key_variant); + } + cfl_variant_destroy(value); + return -1; + } + result = json_cfl_insert(builder, kvlist, key_variant->data.as_string, + cfl_variant_size_get(key_variant), value); + cfl_variant_destroy(key_variant); + return result; +} + +static struct cfl_kvlist *json_converted_kvlist(struct json_cfl_builder *builder, + yyjson_val *array, size_t depth) +{ + size_t index; + size_t maximum; + yyjson_val *entry; + yyjson_val *key; + yyjson_val *value; + struct cfl_kvlist *result; + struct cfl_variant *variant; + + if (!yyjson_is_arr(array)) { + return NULL; + } + if (depth > OTEL_JSON_CFL_MAX_DEPTH) { + builder->unsupported = FLB_TRUE; + return NULL; + } + result = json_cfl_kvlist_create(builder); + if (result == NULL) { + return NULL; + } + yyjson_arr_foreach(array, index, maximum, entry) { + key = json_obj_get(entry, "key"); + if (!yyjson_is_obj(entry) || key == NULL) { + cfl_kvlist_destroy(result); + return NULL; + } + value = json_obj_get(entry, "value"); + if (value == NULL) { + /* an unset value is an empty string */ + variant = json_cfl_string(builder, "", 0); + } + else if (yyjson_is_obj(value) && yyjson_obj_size(value) == 0) { + variant = json_cfl_null(builder); + } + else if (!yyjson_is_obj(value)) { + cfl_kvlist_destroy(result); + return NULL; + } + else { + variant = json_converted_value(builder, value, depth + 1); + } + if (json_converted_insert(builder, result, key, variant, depth) != 0) { + cfl_kvlist_destroy(result); + return NULL; + } + } + return result; +} + +static struct cfl_variant *json_converted_map(struct json_cfl_builder *builder, + yyjson_val *object, size_t depth); + +/* + * Returns 0 with the converted value, NULL on failure, or the json_wrapped() + * status when the object is not a valid wrapper. + */ +static int json_unwrapped_value(struct json_cfl_builder *builder, yyjson_val *object, + size_t depth, struct cfl_variant **out_variant) +{ + int ret; + int type; + size_t length; + size_t bytes_size; + unsigned char *bytes; + char buffer[33]; + yyjson_val *value; + struct cfl_array *array; + + ret = json_wrapped(object, &value, &type); + if (ret != 0) { + return ret; + } + *out_variant = NULL; + + switch (type) { + case JSON_WRAPPED_INT: + if (yyjson_is_str(value)) { + length = yyjson_get_len(value); + if (length > sizeof(buffer) - 1) { + length = sizeof(buffer) - 1; + } + memcpy(buffer, yyjson_get_str(value), length); + buffer[length] = '\0'; + *out_variant = json_cfl_int(builder, strtoll(buffer, NULL, 10)); + return 0; + } + break; + case JSON_WRAPPED_BYTES: + if (json_base64_decode(yyjson_get_str(value), yyjson_get_len(value), + &bytes, &bytes_size) != 0) { + return -2; + } + if (json_cfl_account_sized(builder, json_cfl_binary_header_size(bytes_size), + bytes_size) == 0) { + *out_variant = cfl_variant_create_from_bytes_in(builder->arena, (char *) bytes, + bytes_size, CFL_FALSE); + } + flb_free(bytes); + return 0; + case JSON_WRAPPED_ARRAY: + if (yyjson_is_arr(value)) { + *out_variant = json_converted_array(builder, value, depth + 1); + return 0; + } + if (json_cfl_account(builder, OTEL_JSON_CFL_ARRAY_HEADER_SIZE) != 0) { + return 0; + } + array = cfl_array_create_in(builder->arena, 0); + if (array != NULL) { + *out_variant = cfl_variant_create_from_array_in(builder->arena, array); + if (*out_variant == NULL) { + cfl_array_destroy(array); + } + } + return 0; + case JSON_WRAPPED_KVLIST: + if (yyjson_is_arr(value)) { + *out_variant = json_kvlist_variant(builder, + json_converted_kvlist(builder, value, + depth + 1)); + } + else { + *out_variant = json_converted_map(builder, value, depth + 1); + } + return 0; + default: + break; + } + + /* strings, booleans, numbers and doubles keep their JSON value */ + *out_variant = json_converted_value(builder, value, depth + 1); + return 0; +} + +static struct cfl_variant *json_converted_map(struct json_cfl_builder *builder, + yyjson_val *object, size_t depth) +{ + int ret; + size_t index; + size_t maximum; + yyjson_val *key; + yyjson_val *value; + struct cfl_kvlist *kvlist; + struct cfl_variant *variant; + + if (depth > OTEL_JSON_CFL_MAX_DEPTH) { + builder->unsupported = FLB_TRUE; + return NULL; + } + variant = NULL; + ret = json_unwrapped_value(builder, object, depth, &variant); + if (ret == 0) { + return variant; + } + else if (ret != -1) { + return NULL; + } + + kvlist = json_cfl_kvlist_create(builder); + if (kvlist == NULL) { + return NULL; + } + yyjson_obj_foreach(object, index, maximum, key, value) { + if (json_converted_insert(builder, kvlist, key, + json_converted_value(builder, value, depth + 1), + depth) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + } + return json_kvlist_variant(builder, kvlist); +} + +static struct cfl_variant *json_converted_value(struct json_cfl_builder *builder, + yyjson_val *value, size_t depth) +{ + if (yyjson_is_obj(value)) { + return json_converted_map(builder, value, depth + 1); + } + if (yyjson_is_arr(value)) { + return json_converted_array(builder, value, depth + 1); + } + if (yyjson_is_null(value)) { + /* the established path encodes JSON null as an empty string */ + return json_cfl_string(builder, "", 0); + } + return json_plain_value(builder, value); +} + +/* Copy a JSON value as is, the way the established path copies msgpack objects */ +static struct cfl_variant *json_raw_value(struct json_cfl_builder *builder, + yyjson_val *value, size_t depth) +{ + size_t index; + size_t maximum; + yyjson_val *key; + yyjson_val *entry; + struct cfl_array *array; + struct cfl_kvlist *kvlist; + struct cfl_variant *variant; + + if (depth > OTEL_JSON_CFL_MAX_DEPTH) { + return NULL; + } + if (yyjson_is_obj(value)) { + if (json_cfl_account(builder, + json_cfl_array_header_size(yyjson_obj_size(value))) != 0) { + return NULL; + } + kvlist = cfl_kvlist_create_in(builder->arena); + if (kvlist == NULL) { + return NULL; + } + yyjson_obj_foreach(value, index, maximum, key, entry) { + if (json_cfl_insert(builder, kvlist, yyjson_get_str(key), yyjson_get_len(key), + json_raw_value(builder, entry, depth + 1)) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + } + variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); + if (variant == NULL) { + cfl_kvlist_destroy(kvlist); + } + return variant; + } + if (yyjson_is_arr(value)) { + if (json_cfl_account(builder, + json_cfl_array_header_size(yyjson_arr_size(value))) != 0) { + return NULL; + } + array = cfl_array_create_in(builder->arena, yyjson_arr_size(value)); + if (array == NULL) { + return NULL; + } + yyjson_arr_foreach(value, index, maximum, entry) { + variant = json_raw_value(builder, entry, depth + 1); + if (variant == NULL || cfl_array_append(array, variant) != 0) { + if (variant != NULL) { + cfl_variant_destroy(variant); + } + cfl_array_destroy(array); + return NULL; + } + } + variant = cfl_variant_create_from_array_in(builder->arena, array); + if (variant == NULL) { + cfl_array_destroy(array); + } + return variant; + } + return json_plain_value(builder, value); +} + +/* + * Parse a decimal string the way the established path does: at most 31 + * characters are considered, and they must all be digits. Values beyond + * 64 bits saturate. Returns the number of digits, or -1. + */ +static int json_parse_digits(yyjson_val *value, uint64_t *result) +{ + size_t index; + size_t length; + const char *string; + char buffer[32]; + + if (!yyjson_is_str(value)) { + return -1; + } + string = yyjson_get_str(value); + length = yyjson_get_len(value); + if (length > sizeof(buffer) - 1) { + length = sizeof(buffer) - 1; + } + for (index = 0; index < length && string[index] != '\0'; index++) { + if (!isdigit((unsigned char) string[index])) { + return -1; + } + buffer[index] = string[index]; + } + buffer[index] = '\0'; + *result = index > 0 ? strtoull(buffer, NULL, 10) : 0; + return index; +} + +/* The timestamp of a log record, it must be an unsigned integer or digits */ +static int json_record_timestamp(yyjson_val *value, struct flb_time *tm) +{ + uint64_t number; + + if (yyjson_is_uint(value)) { + number = yyjson_get_uint(value); + } + else if (json_parse_digits(value, &number) < 0) { + return -1; + } + return flb_time_from_uint64(tm, number); +} + +/* A non-empty identifier must have the exact hexadecimal length */ +static int json_identifier_valid(yyjson_val *value, size_t length) +{ + size_t index; + const char *string; + + if (!yyjson_is_str(value)) { + return FLB_FALSE; + } + if (yyjson_get_len(value) == 0) { + return FLB_TRUE; + } + if (yyjson_get_len(value) != length) { + return FLB_FALSE; + } + string = yyjson_get_str(value); + for (index = 0; index < length; index++) { + if (!isxdigit((unsigned char) string[index])) { + return FLB_FALSE; + } + } + return FLB_TRUE; +} + +/* + * Reject a log record the established path rejects, with the same status and + * in the same order, before any of it is built. + */ +static int json_record_validate(struct json_cfl_builder *builder, yyjson_val *log_record, + struct flb_time *tm) +{ + yyjson_val *value; + + if (!yyjson_is_obj(log_record)) { + return FLB_OTEL_LOGS_ERR_UNEXPECTED_LOGRECORDS_ENTRY_TYPE; + } + value = json_obj_get(log_record, "timeUnixNano"); + if (value == NULL) { + value = json_obj_get(log_record, "observedTimeUnixNano"); + } + if (value == NULL) { + flb_time_get(tm); + } + else if (json_record_timestamp(value, tm) != 0) { + return FLB_OTEL_LOGS_ERR_UNEXPECTED_TIMESTAMP_TYPE; + } + else if (flb_time_is_valid_eventtime(tm) != FLB_TRUE) { + /* the msgpack path keeps such a record with an unrelated timestamp */ + builder->unsupported = FLB_TRUE; + return FLB_OTEL_LOGS_ERR_UNEXPECTED_TIMESTAMP_TYPE; + } + value = json_obj_get(log_record, "attributes"); + if (value != NULL && !yyjson_is_arr(value)) { + return FLB_OTEL_LOGS_ERR_UNEXPECTED_ATTRIBUTES_TYPE; + } + value = json_obj_get(log_record, "traceId"); + if (value != NULL && json_identifier_valid(value, 32) == FLB_FALSE) { + return FLB_OTEL_LOGS_ERR_INVALID_TRACE_ID; + } + value = json_obj_get(log_record, "spanId"); + if (value != NULL && json_identifier_valid(value, 16) == FLB_FALSE) { + return FLB_OTEL_LOGS_ERR_INVALID_SPAN_ID; + } + value = json_obj_get(log_record, "body"); + if (value != NULL && !yyjson_is_obj(value)) { + return FLB_OTEL_LOGS_ERR_UNEXPECTED_BODY_TYPE; + } + return 0; +} + +static struct cfl_variant *json_plain_value(struct json_cfl_builder *builder, + yyjson_val *value) +{ + if (yyjson_is_str(value)) { + return json_cfl_string(builder, yyjson_get_str(value), yyjson_get_len(value)); + } + if (yyjson_is_uint(value)) { + return json_cfl_uint(builder, yyjson_get_uint(value)); + } + if (yyjson_is_sint(value)) { + return json_cfl_int(builder, yyjson_get_sint(value)); + } + if (yyjson_is_real(value)) { + return json_cfl_double(builder, yyjson_get_real(value)); + } + if (yyjson_is_bool(value)) { + return json_cfl_bool(builder, yyjson_get_bool(value)); + } + if (yyjson_is_null(value)) { + return json_cfl_null(builder); + } + return NULL; +} + +static struct cfl_kvlist *json_record_metadata(struct json_cfl_builder *builder, + yyjson_val *log_record) +{ + int result; + uint64_t number; + unsigned char identifier[16]; + yyjson_val *value; + struct cfl_kvlist *metadata; + struct cfl_kvlist *otlp; + + result = 0; + metadata = json_cfl_kvlist_create(builder); + otlp = json_cfl_kvlist_create(builder); + if (metadata == NULL || otlp == NULL) { + goto error; + } + value = json_obj_get(log_record, "observedTimeUnixNano"); + if (json_parse_digits(value, &number) > 0) { + /* the established path stores it as a signed integer */ + result = json_cfl_insert(builder, otlp, "observed_timestamp", 18, + json_cfl_int(builder, (int64_t) number)); + } + value = json_obj_get(log_record, "severityNumber"); + if (result == 0 && value != NULL) { + result = json_cfl_insert(builder, otlp, "severity_number", 15, + json_raw_value(builder, value, 0)); + } + value = json_obj_get(log_record, "severityText"); + if (result == 0 && yyjson_is_str(value)) { + result = json_cfl_insert(builder, otlp, "severity_text", 13, + json_cfl_string(builder, yyjson_get_str(value), + yyjson_get_len(value))); + } + value = json_obj_get(log_record, "attributes"); + if (result == 0 && value != NULL) { + result = json_cfl_insert_kvlist(builder, otlp, "attributes", 10, + json_converted_kvlist(builder, value, 0)); + if (result != 0) { + builder->error_status = FLB_OTEL_LOGS_ERR_UNEXPECTED_ATTRIBUTES_TYPE; + } + } + value = json_obj_get(log_record, "droppedAttributesCount"); + if (result == 0 && value != NULL && (yyjson_is_int(value) || yyjson_is_uint(value))) { + result = json_cfl_insert(builder, otlp, "dropped_attributes_count", 24, + json_plain_value(builder, value)); + } + value = json_obj_get(log_record, "traceId"); + if (result == 0 && yyjson_is_str(value) && yyjson_get_len(value) > 0) { + if (yyjson_get_len(value) != 32 || + flb_otel_utils_hex_to_id(yyjson_get_str(value), 32, identifier, 16) != 0 || + json_cfl_account_sized(builder, json_cfl_binary_header_size(16), 16) != 0) { + goto error; + } + result = json_cfl_insert(builder, otlp, "trace_id", 8, + cfl_variant_create_from_bytes_in(builder->arena, + (char *) identifier, 16, CFL_FALSE)); + } + value = json_obj_get(log_record, "spanId"); + if (result == 0 && yyjson_is_str(value) && yyjson_get_len(value) > 0) { + if (yyjson_get_len(value) != 16 || + flb_otel_utils_hex_to_id(yyjson_get_str(value), 16, identifier, 8) != 0 || + json_cfl_account_sized(builder, json_cfl_binary_header_size(8), 8) != 0) { + goto error; + } + result = json_cfl_insert(builder, otlp, "span_id", 7, + cfl_variant_create_from_bytes_in(builder->arena, + (char *) identifier, 8, CFL_FALSE)); + } + value = json_obj_get(log_record, "flags"); + if (value == NULL) { + value = json_obj_get(log_record, "traceFlags"); + } + if (result == 0 && (yyjson_is_uint(value) || json_parse_digits(value, &number) > 0)) { + if (yyjson_is_uint(value)) { + number = yyjson_get_uint(value); + } + result = json_cfl_insert(builder, otlp, "trace_flags", 11, + json_cfl_uint(builder, number & 0xff)); + } + value = json_obj_get(log_record, "eventName"); + if (result == 0 && yyjson_is_str(value)) { + result = json_cfl_insert(builder, otlp, "event_name", 10, + json_cfl_string(builder, yyjson_get_str(value), + yyjson_get_len(value))); + } + if (result == 0) { + result = json_cfl_insert_kvlist(builder, metadata, "otlp", 4, otlp); + otlp = NULL; + } + if (result == 0) { + return metadata; + } + +error: + cfl_kvlist_destroy(otlp); + cfl_kvlist_destroy(metadata); + return NULL; +} + +static struct cfl_kvlist *json_record_body(struct json_cfl_builder *builder, + yyjson_val *log_record) +{ + int type; + const char *key; + yyjson_val *body; + struct cfl_kvlist *result; + struct cfl_variant *variant; + + body = json_obj_get(log_record, "body"); + if (body == NULL) { + return json_cfl_kvlist_create(builder); + } + + /* + * A kvlist, or a map that is not a wrapped value, is the record itself. + * Anything else is stored under the body key. + */ + type = -1; + if (json_wrapped(body, NULL, &type) != 0 || + (builder->options->body_key == NULL && type == JSON_WRAPPED_KVLIST)) { + variant = json_converted_value(builder, body, 0); + if (variant == NULL || variant->type != CFL_VARIANT_KVLIST) { + if (variant != NULL) { + cfl_variant_destroy(variant); + } + builder->error_status = FLB_OTEL_LOGS_ERR_APPEND_BODY_FAILURE; + return NULL; + } + result = variant->data.as_kvlist; + variant->data.as_kvlist = NULL; + variant->type = CFL_VARIANT_NULL; + cfl_variant_destroy(variant); + return result; + } + + result = json_cfl_kvlist_create(builder); + if (result == NULL) { + return NULL; + } + key = builder->options->body_key != NULL ? builder->options->body_key : "log"; + variant = json_converted_value(builder, body, 0); + if (json_cfl_insert(builder, result, key, strlen(key), variant) != 0) { + builder->error_status = FLB_OTEL_LOGS_ERR_APPEND_BODY_FAILURE; + cfl_kvlist_destroy(result); + return NULL; + } + return result; +} + +static struct cfl_kvlist *json_group_metadata(struct json_cfl_builder *builder, + size_t resource_index, size_t scope_index) +{ + struct cfl_kvlist *result; + + result = json_cfl_kvlist_create(builder); + if (result == NULL || + json_cfl_insert(builder, result, "schema", 6, + json_cfl_string(builder, "otlp", 4)) != 0 || + json_cfl_insert(builder, result, "resource_id", 11, + json_cfl_uint(builder, resource_index)) != 0 || + json_cfl_insert(builder, result, "scope_id", 8, + json_cfl_uint(builder, scope_index)) != 0) { + cfl_kvlist_destroy(result); + return NULL; + } + return result; +} + +static struct cfl_kvlist *json_group_body(struct json_cfl_builder *builder, + yyjson_val *resource_log, + yyjson_val *scope_log) +{ + int result; + yyjson_val *resource; + yyjson_val *scope; + yyjson_val *schema_url; + yyjson_val *value; + struct cfl_kvlist *body; + struct cfl_kvlist *resource_map; + struct cfl_kvlist *scope_map; + + result = 0; + scope_map = NULL; + body = json_cfl_kvlist_create(builder); + resource_map = json_cfl_kvlist_create(builder); + if (body == NULL || resource_map == NULL) { + goto error; + } + resource = json_obj_get(resource_log, "resource"); + if (yyjson_is_obj(resource)) { + value = json_obj_get(resource, "attributes"); + if (value != NULL) { + result = json_cfl_insert_kvlist(builder, resource_map, "attributes", 10, + json_converted_kvlist(builder, value, 0)); + if (result != 0) { + builder->error_status = FLB_OTEL_RESOURCE_INVALID_ATTRIBUTE; + } + } + value = json_obj_get(resource, "droppedAttributesCount"); + if (result == 0 && value != NULL) { + result = json_cfl_insert(builder, resource_map, "dropped_attributes_count", 24, + json_raw_value(builder, value, 0)); + } + } + value = json_obj_get(resource_log, "schemaUrl"); + if (result == 0 && yyjson_is_str(value)) { + result = json_cfl_insert(builder, resource_map, "schema_url", 10, + json_cfl_string(builder, yyjson_get_str(value), + yyjson_get_len(value))); + } + if (result == 0) { + result = json_cfl_insert_kvlist(builder, body, "resource", 8, resource_map); + resource_map = NULL; + } + scope = json_obj_get(scope_log, "scope"); + if (!yyjson_is_obj(scope)) { + scope = NULL; + } + schema_url = json_obj_get(scope_log, "schemaUrl"); + if (!yyjson_is_str(schema_url)) { + schema_url = NULL; + } + + /* the scope is only described when the request carries one */ + if (scope == NULL && schema_url == NULL) { + if (result == 0) { + return body; + } + goto error; + } + if (result == 0) { + scope_map = json_cfl_kvlist_create(builder); + if (scope_map == NULL) { + goto error; + } + } + if (result == 0 && scope != NULL) { + value = json_obj_get(scope, "name"); + if (yyjson_is_str(value)) { + result = json_cfl_insert(builder, scope_map, "name", 4, + json_cfl_string(builder, yyjson_get_str(value), + yyjson_get_len(value))); + } + value = json_obj_get(scope, "version"); + if (result == 0 && yyjson_is_str(value)) { + result = json_cfl_insert(builder, scope_map, "version", 7, + json_cfl_string(builder, yyjson_get_str(value), + yyjson_get_len(value))); + } + value = json_obj_get(scope, "droppedAttributesCount"); + if (result == 0 && value != NULL) { + result = json_cfl_insert(builder, scope_map, "dropped_attributes_count", 24, + json_raw_value(builder, value, 0)); + } + value = json_obj_get(scope, "attributes"); + if (result == 0 && value != NULL) { + result = json_cfl_insert_kvlist(builder, scope_map, "attributes", 10, + json_converted_kvlist(builder, value, 0)); + if (result != 0) { + builder->error_status = FLB_OTEL_LOGS_ERR_SCOPE_KVLIST; + } + } + } + if (result == 0 && schema_url != NULL) { + result = json_cfl_insert(builder, scope_map, "schema_url", 10, + json_cfl_string(builder, yyjson_get_str(schema_url), + yyjson_get_len(schema_url))); + } + if (result == 0) { + result = json_cfl_insert_kvlist(builder, body, "scope", 5, scope_map); + scope_map = NULL; + } + if (result == 0) { + return body; + } + +error: + cfl_kvlist_destroy(scope_map); + cfl_kvlist_destroy(resource_map); + cfl_kvlist_destroy(body); + return NULL; +} + +static int json_scope_logs(struct json_cfl_builder *builder, + yyjson_val *resource_log, yyjson_val *scope_log, + size_t resource_index, size_t scope_index, + size_t *record_count) +{ + size_t index; + size_t maximum; + yyjson_val *records; + yyjson_val *log_record; + struct flb_time tm; + struct flb_mp_chunk_record *group; + struct flb_mp_chunk_record *record; + + records = json_obj_get(scope_log, "logRecords"); + if (records == NULL || (yyjson_is_arr(records) && yyjson_arr_size(records) == 0)) { + return 0; + } + flb_time_set(&tm, FLB_LOG_EVENT_GROUP_START, 0); + group = json_cfl_record_add(builder, &tm); + if (group == NULL) { + return -1; + } + group->cobj_metadata = json_cfl_object(builder, + json_group_metadata(builder, resource_index, scope_index)); + group->cobj_record = json_cfl_object(builder, + json_group_body(builder, resource_log, scope_log)); + if (group->cobj_metadata == NULL || group->cobj_record == NULL) { + return -1; + } + if (!yyjson_is_arr(records)) { + builder->error_status = FLB_OTEL_LOGS_ERR_UNEXPECTED_LOGRECORDS_TYPE; + return -1; + } + group->cobj_group_metadata = group->cobj_metadata; + group->cobj_group_attributes = group->cobj_record; + group->owns_group_metadata = FLB_TRUE; + group->owns_group_attributes = FLB_TRUE; + + yyjson_arr_foreach(records, index, maximum, log_record) { + builder->error_status = json_record_validate(builder, log_record, &tm); + if (builder->error_status != 0) { + return -1; + } + record = json_cfl_record_add(builder, &tm); + if (record == NULL) { + return -1; + } + /* the metadata is checked first, it decides the status of a bad record */ + record->cobj_metadata = json_cfl_object(builder, + json_record_metadata(builder, log_record)); + if (record->cobj_metadata == NULL) { + return -1; + } + record->cobj_record = json_cfl_object(builder, json_record_body(builder, log_record)); + if (record->cobj_record == NULL) { + return -1; + } + record->cobj_group_metadata = group->cobj_group_metadata; + record->cobj_group_attributes = group->cobj_group_attributes; + (*record_count)++; + } + + flb_time_set(&tm, FLB_LOG_EVENT_GROUP_END, 0); + record = json_cfl_record_add(builder, &tm); + if (record == NULL) { + return -1; + } + record->cobj_metadata = json_cfl_object(builder, json_cfl_kvlist_create(builder)); + record->cobj_record = json_cfl_object(builder, json_cfl_kvlist_create(builder)); + if (record->cobj_metadata == NULL || record->cobj_record == NULL) { + return -1; + } + record->cobj_group_metadata = group->cobj_group_metadata; + record->cobj_group_attributes = group->cobj_group_attributes; + return 0; +} + +int flb_opentelemetry_logs_json_to_cfl( + const char *body, size_t length, + const struct flb_opentelemetry_logs_decode_options *options, + struct flb_opentelemetry_logs_json_cfl **out_result, + int *error_status) +{ + int result; + int status; + size_t resource_index; + size_t resource_maximum; + size_t scope_index; + size_t scope_maximum; + yyjson_val *root; + yyjson_val *resource_logs; + yyjson_val *resource_log; + yyjson_val *scope_logs; + yyjson_val *scope_log; + yyjson_val *resource; + struct json_cfl_builder builder; + struct flb_opentelemetry_logs_json_cfl *output; + + status = 0; + builder.error_status = 0; + builder.unsupported = FLB_FALSE; + + if (error_status != NULL) { + *error_status = 0; + } + if (body == NULL || options == NULL || options->metadata_key == NULL || + out_result == NULL) { + return -1; + } + *out_result = NULL; + output = flb_calloc(1, sizeof(*output)); + if (output == NULL) { + return -1; + } + output->arena = json_arena_create(); + output->document = yyjson_read(body, length, 0); + if (output->arena == NULL || output->document == NULL) { + result = -1; + goto error; + } + root = yyjson_doc_get_root(output->document); + if (!yyjson_is_obj(root)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_ROOT_OBJECT_TYPE; + result = -1; + goto error; + } + resource_logs = json_obj_get(root, "resourceLogs"); + if (resource_logs == NULL) { + status = FLB_OTEL_LOGS_ERR_RESOURCELOGS_MISSING; + result = -1; + goto error; + } + if (!yyjson_is_arr(resource_logs)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_RESOURCELOGS_TYPE; + result = -1; + goto error; + } + builder.options = options; + builder.arena = output->arena; + builder.size = 0; + builder.limit = options->output_limit; + builder.chunk = flb_mp_chunk_cobj_create(NULL, NULL); + if (builder.chunk == NULL) { + result = -1; + goto error; + } + builder.chunk->record_pool_enabled = FLB_TRUE; + output->chunk = builder.chunk; + + /* the checks and their order follow flb_opentelemetry_logs_json_to_msgpack() */ + yyjson_arr_foreach(resource_logs, resource_index, resource_maximum, resource_log) { + if (!yyjson_is_obj(resource_log)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_RESOURCELOGS_ENTRY_TYPE; + result = -1; + goto error; + } + resource = json_obj_get(resource_log, "resource"); + if (yyjson_is_obj(resource) && json_obj_get(resource, "attributes") != NULL && + !yyjson_is_arr(json_obj_get(resource, "attributes"))) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_ATTRIBUTES_TYPE; + result = -1; + goto error; + } + scope_logs = json_obj_get(resource_log, "scopeLogs"); + if (scope_logs == NULL) { + status = FLB_OTEL_LOGS_ERR_SCOPELOGS_MISSING; + result = -1; + goto error; + } + if (!yyjson_is_arr(scope_logs)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_SCOPELOGS_TYPE; + result = -1; + goto error; + } + yyjson_arr_foreach(scope_logs, scope_index, scope_maximum, scope_log) { + if (!yyjson_is_obj(scope_log)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_SCOPELOGS_TYPE; + result = -1; + goto error; + } + if (json_scope_logs(&builder, resource_log, scope_log, + resource_index, scope_index, + &output->record_count) != 0) { + status = builder.error_status; + result = -1; + if (builder.unsupported == FLB_TRUE) { + status = 0; + result = FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED; + } + goto error; + } + } + } + output->encoded_size = builder.size; + *out_result = output; + return 0; + +error: + if (error_status != NULL) { + *error_status = status; + } + flb_opentelemetry_logs_json_cfl_destroy(output); + return result; +} + +struct flb_mp_chunk_cobj *flb_opentelemetry_logs_json_cfl_take_chunk( + struct flb_opentelemetry_logs_json_cfl *result) +{ + struct flb_mp_chunk_cobj *chunk; + + if (result == NULL) { + return NULL; + } + chunk = result->chunk; + result->chunk = NULL; + return chunk; +} + +size_t flb_opentelemetry_logs_json_cfl_get_encoded_size( + struct flb_opentelemetry_logs_json_cfl *result) +{ + return result != NULL ? result->encoded_size : 0; +} + +void flb_opentelemetry_logs_json_cfl_destroy( + struct flb_opentelemetry_logs_json_cfl *result) +{ + if (result == NULL) { + return; + } + flb_mp_chunk_cobj_destroy(result->chunk); + cfl_arena_destroy(result->arena); + yyjson_doc_free(result->document); + flb_free(result); +} From 6d230fd30c882ce212ed5a0a35356333bd08ebe4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 06:19:42 -0600 Subject: [PATCH 17/33] in_opentelemetry: enable direct CFL for JSON logs Decode JSON logs with the single-pass CFL decoder: as CFL records for native processor chains, encoded once to msgpack otherwise. A request the CFL decoder cannot hold, such as an attribute with a non-string key, uses the msgpack conversion. Signed-off-by: Eduardo Silva --- plugins/in_opentelemetry/opentelemetry_logs.c | 98 ++++++++++++++++--- 1 file changed, 82 insertions(+), 16 deletions(-) diff --git a/plugins/in_opentelemetry/opentelemetry_logs.c b/plugins/in_opentelemetry/opentelemetry_logs.c index 9e6792b8116..2508d43504e 100644 --- a/plugins/in_opentelemetry/opentelemetry_logs.c +++ b/plugins/in_opentelemetry/opentelemetry_logs.c @@ -29,6 +29,51 @@ #include "opentelemetry_logs.h" #include "opentelemetry_utils.h" +/* + * Encode JSON logs through the single-pass CFL decoder. The generic JSON to + * msgpack path parses the payload into msgpack first and then walks that + * msgpack again to build the records, it is only used for the rare payloads + * CFL cannot hold. + */ +static int opentelemetry_logs_json_encode_direct(struct flb_opentelemetry *ctx, + struct flb_log_event_encoder *encoder, + const char *payload, size_t size, + struct flb_opentelemetry_logs_decode_options *options, + size_t *record_count, + int *error_status) +{ + int ret; + struct flb_mp_chunk_cobj *chunk; + struct flb_opentelemetry_logs_json_cfl *result; + + result = NULL; + ret = flb_opentelemetry_logs_json_to_cfl(payload, size, options, &result, error_status); + if (ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED) { + return flb_opentelemetry_logs_json_to_msgpack(encoder, payload, size, + ctx->logs_body_key, error_status); + } + if (ret != 0) { + flb_opentelemetry_logs_json_cfl_destroy(result); + return -1; + } + + chunk = flb_opentelemetry_logs_json_cfl_take_chunk(result); + if (chunk == NULL) { + flb_opentelemetry_logs_json_cfl_destroy(result); + return -1; + } + + /* the records borrow strings from the JSON document owned by 'result' */ + chunk->log_encoder = encoder; + ret = flb_mp_chunk_cobj_encode_append(chunk); + *record_count = flb_mp_chunk_cobj_count_log_records(chunk); + chunk->log_encoder = NULL; + + flb_mp_chunk_cobj_destroy(chunk); + flb_opentelemetry_logs_json_cfl_destroy(result); + + return ret; +} /* * Main function used from opentelemetry_prot.c to process logs either in JSON or Protobuf format. @@ -63,6 +108,7 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, struct flb_mp_chunk_cobj *chunk; struct flb_opentelemetry_logs_decode_options decode_options; struct flb_opentelemetry_logs_protobuf_cfl *cfl_result; + struct flb_opentelemetry_logs_json_cfl *json_cfl_result; int worker_thread; buf = (char *) data; @@ -70,6 +116,7 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, payload_size = size; record_count = 0; cfl_result = NULL; + json_cfl_result = NULL; worker_thread = opentelemetry_uses_worker_ingress_queue(ctx); decode_options.metadata_key = ctx->logs_metadata_key; decode_options.body_key = ctx->logs_body_key; @@ -94,22 +141,41 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, } /* - * Protobuf records go straight to the input processors as CFL records when - * they can take them. Worker threads enqueue the serialized processor + * JSON and protobuf records go straight to input processors as CFL records + * when they can take them. Workers enqueue the serialized processor * output as already processed, so the engine does not run the chain twice. + * A JSON payload CFL cannot hold takes the msgpack path below. */ - if (is_proto == FLB_TRUE && - flb_input_log_chunk_supported(ctx->ins, worker_thread) == FLB_TRUE) { - ret = flb_opentelemetry_logs_protobuf_to_cfl( - payload, payload_size, &decode_options, &cfl_result); + if (flb_input_log_chunk_supported(ctx->ins, worker_thread) == FLB_TRUE) { + if (is_proto == FLB_TRUE) { + ret = flb_opentelemetry_logs_protobuf_to_cfl( + payload, payload_size, &decode_options, &cfl_result); + if (ret == 0) { + *remaining -= flb_opentelemetry_logs_protobuf_cfl_get_encoded_size(cfl_result); + chunk = flb_opentelemetry_logs_protobuf_cfl_take_chunk(cfl_result); + } + } + else { + ret = flb_opentelemetry_logs_json_to_cfl( + (const char *) payload, payload_size, &decode_options, + &json_cfl_result, &error_status); + if (ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED) { + goto msgpack_path; + } + if (ret == 0) { + *remaining -= flb_opentelemetry_logs_json_cfl_get_encoded_size(json_cfl_result); + chunk = flb_opentelemetry_logs_json_cfl_take_chunk(json_cfl_result); + } + } if (ret != 0) { - flb_plg_error(ctx->ins, "failed to process logs from protobuf payload"); + flb_plg_error(ctx->ins, "failed to process logs from %s payload (%i) %s", + is_proto == FLB_TRUE ? "protobuf" : "JSON", error_status, + flb_opentelemetry_error_to_string(error_status)); + flb_opentelemetry_logs_protobuf_cfl_destroy(cfl_result); + flb_opentelemetry_logs_json_cfl_destroy(json_cfl_result); return -1; } - *remaining -= flb_opentelemetry_logs_protobuf_cfl_get_encoded_size(cfl_result); - chunk = flb_opentelemetry_logs_protobuf_cfl_take_chunk(cfl_result); - /* like the Collector, a request without log records is accepted */ if (cfl_list_is_empty(&chunk->records)) { flb_mp_chunk_cobj_destroy(chunk); @@ -120,6 +186,7 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, chunk, worker_thread); } flb_opentelemetry_logs_protobuf_cfl_destroy(cfl_result); + flb_opentelemetry_logs_json_cfl_destroy(json_cfl_result); if (ret != 0) { flb_plg_error(ctx->ins, "failed to append logs to the input buffer"); return -1; @@ -128,6 +195,7 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, return 0; } +msgpack_path: encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); if (encoder == NULL) { return -1; @@ -142,17 +210,15 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, } } else { - ret = flb_opentelemetry_logs_json_to_msgpack(encoder, - (const char *) payload, payload_size, - ctx->logs_body_key, - &error_status); + ret = opentelemetry_logs_json_encode_direct(ctx, encoder, + (const char *) payload, payload_size, + &decode_options, &record_count, + &error_status); if (ret != 0) { - /* we are printing the error for now, let's see what is the user's preference later */ flb_plg_error(ctx->ins, "failed to process logs from JSON payload (%i) %s", error_status, flb_opentelemetry_error_to_string(error_status)); } - } if (ret >= 0 && encoder->output_length == 0) { From aea7fd73ebe270c523964423f90bc414976719e0 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 06:19:42 -0600 Subject: [PATCH 18/33] tests: integration: cover direct OTLP JSON logs Signed-off-by: Eduardo Silva --- .../tests/test_in_opentelemetry_001.py | 184 ++++++++++++++++-- 1 file changed, 173 insertions(+), 11 deletions(-) diff --git a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py index 20fb3bdbf7b..42684fcfa75 100644 --- a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py +++ b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py @@ -2737,6 +2737,10 @@ def test_in_opentelemetry_http_snappy_invalid_checksum(frame_type): {"name": "content_modifier", "context": "attributes", "action": "hash", "key": "marker"}, ], + "rename": [ + {"name": "content_modifier", "context": "body", + "action": "rename", "key": "marker", "value": "renamed_marker"}, + ], "hash_then_sql": [ {"name": "content_modifier", "context": "attributes", "action": "hash", "key": "marker"}, @@ -2864,6 +2868,9 @@ def direct_cfl_service(tmp_path, direct, processors, *, body_key=None, workers=1 if threaded: options.append("threaded: true") option_lines = "\n".join(f" {option}" for option in options) + processor_lines = "" + if processors is not None: + processor_lines = f" processors: {json.dumps({'logs': processors})}\n" route_lines = "" output_match = " match: '*'" if conditional_routes: @@ -2891,8 +2898,7 @@ def direct_cfl_service(tmp_path, direct, processors, *, body_key=None, workers=1 port: ${{FLUENT_BIT_TEST_LISTENER_PORT}} http2: on {option_lines} - processors: {json.dumps({"logs": processors})} -{route_lines} outputs: +{processor_lines}{route_lines} outputs: - name: opentelemetry alias: direct_cfl_output {output_match} @@ -2977,12 +2983,151 @@ def test_in_opentelemetry_logs_direct_cfl_parity(tmp_path, chain, transport): assert direct_records == msgpack_records -def test_in_opentelemetry_logs_direct_cfl_body_key(tmp_path): +@pytest.mark.parametrize("chain", sorted(DIRECT_CFL_PROCESSOR_CHAINS)) +def test_in_opentelemetry_logs_direct_cfl_json_parity(tmp_path, chain): + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + msgpack_records = run_direct_cfl_case( + tmp_path, False, processors, "http", encoding="json") + direct_records = run_direct_cfl_case( + tmp_path, True, processors, "http", encoding="json") + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.parametrize("workers", [1, pytest.param( + 4, marks=pytest.mark.skipif(os.name == "nt", + reason="HTTP shared listener ports are unavailable on Windows"))]) +def test_in_opentelemetry_logs_direct_cfl_json_without_processors(tmp_path, workers): + # Without processors JSON uses the single-pass CFL decoder and is encoded + # once, the records must match the msgpack reference. + msgpack_records = run_direct_cfl_case( + tmp_path, False, None, "http", encoding="json", workers=workers) + direct_records = run_direct_cfl_case( + tmp_path, True, None, "http", encoding="json", workers=workers) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.parametrize("processors", [None, "content_modifier"]) +def test_in_opentelemetry_logs_direct_cfl_json_edge_parity(tmp_path, processors): + if processors is not None: + processors = DIRECT_CFL_PROCESSOR_CHAINS[processors] + payload = { + "resourceLogs": [{ + "resource": {"attributes": [ + {"key": "missing"}, + {"key": "unset", "value": {}}, + {"key": "nested", "value": { + "customValue": {"child": [1, True, None]} + }}, + ]}, + "scopeLogs": [{"logRecords": [ + {"timeUnixNano": "1789516800123456789", + "body": {"arrayValue": {"values": [ + {}, {"stringValue": "value"} + ]}}}, + {"timeUnixNano": "1789516800123456790", "body": {}}, + ]}], + }] + } + results = [] + + for direct in [False, True]: + service = direct_cfl_service(tmp_path, direct, processors) + try: + service.start() + before = len(data_storage["logs"]) + response = service.send_raw_request( + "/v1/logs", json.dumps(payload).encode("utf-8"), + content_type="application/json") + assert response.status_code == 201 + + def received_records(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if len(records) == 2 else None + + received = service.service.wait_for_condition( + received_records, + timeout=20, interval=0.1, description="direct CFL JSON edge records") + results.append(received) + finally: + service.stop() + + assert results[1] == results[0] + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +def test_in_opentelemetry_logs_direct_cfl_empty_request(tmp_path, encoding): + # A request without log records is valid and accepted on every path. + if encoding == "json": + payload = b'{"resourceLogs":[{"scopeLogs":[{"logRecords":[]}]}]}' + content_type = "application/json" + else: + request = ExportLogsServiceRequest() + request.resource_logs.add().scope_logs.add() + payload = request.SerializeToString() + content_type = "application/x-protobuf" + + for direct in [False, True]: + service = direct_cfl_service(tmp_path, direct, + DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"]) + try: + service.start() + response = service.send_raw_request("/v1/logs", payload, + content_type=content_type) + assert response.status_code == 201 + finally: + service.stop() + + +@pytest.mark.parametrize("processors", [None, "content_modifier"]) +def test_in_opentelemetry_logs_direct_cfl_json_unsupported_key(tmp_path, processors): + # CFL maps only hold string keys. A wrapped attribute key is unwrapped, + # a numeric one makes the request take the msgpack path. + if processors is not None: + processors = DIRECT_CFL_PROCESSOR_CHAINS[processors] + payload = {"resourceLogs": [{"scopeLogs": [{"logRecords": [ + {"timeUnixNano": "1789516800000000000", "body": {"stringValue": "wrapped"}, + "attributes": [{"key": {"stringValue": "name"}, "value": {"intValue": "1"}}]}, + {"timeUnixNano": "1789516800000000001", "body": {"stringValue": "numeric"}, + "attributes": [{"key": 5, "value": {"boolValue": True}}]}, + ]}]}]} + sentinel = build_direct_cfl_sentinel_payload().SerializeToString() + results = [] + + for direct in [False, True]: + service = direct_cfl_service(tmp_path, direct, processors) + try: + service.start() + before = len(data_storage["logs"]) + response = service.send_raw_request( + "/v1/logs", json.dumps(payload).encode("utf-8"), + content_type="application/json") + assert response.status_code == 201 + response = service.send_raw_request("/v1/logs", sentinel) + assert response.status_code == 201 + + def received(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if any(is_sentinel(record) for record in records) else None + + results.append(service.service.wait_for_condition( + received, timeout=20, interval=0.1, description="sentinel after key test")) + finally: + service.stop() + + assert results[1] == results[0] + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +def test_in_opentelemetry_logs_direct_cfl_body_key(tmp_path, encoding): processors = DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"] msgpack_records = run_direct_cfl_case(tmp_path, False, processors, "http", - body_key="message") + body_key="message", encoding=encoding) direct_records = run_direct_cfl_case(tmp_path, True, processors, "http", - body_key="message") + body_key="message", encoding=encoding) assert len(msgpack_records) > 1 assert direct_records == msgpack_records @@ -3098,14 +3243,18 @@ def test_in_opentelemetry_logs_direct_cfl_workers_fallback(tmp_path, chain): @pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) @pytest.mark.parametrize("chain", ["content_modifier", "sql_drop", "hash_then_sql", "conditional_safe"]) -def test_in_opentelemetry_logs_direct_cfl_threaded_worker_safe_chain(tmp_path, chain): +def test_in_opentelemetry_logs_direct_cfl_threaded_worker_safe_chain(tmp_path, chain, + encoding): processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] msgpack_records = run_direct_cfl_case( - tmp_path, False, processors, "http", workers=4, threaded=True) + tmp_path, False, processors, "http", workers=4, threaded=True, + encoding=encoding) direct_records = run_direct_cfl_case( - tmp_path, True, processors, "http", workers=4, threaded=True) + tmp_path, True, processors, "http", workers=4, threaded=True, + encoding=encoding) assert len(msgpack_records) > 1 assert direct_records == msgpack_records @@ -3161,10 +3310,22 @@ def received_records(): service.stop() +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) @pytest.mark.parametrize("limit", [512, 1024, 2048, 4096, 8192, 16384]) -def test_in_opentelemetry_logs_direct_cfl_limit_parity(tmp_path, limit): +def test_in_opentelemetry_logs_direct_cfl_limit_parity(tmp_path, limit, encoding): processors = DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"] - payload = build_direct_cfl_logs_payload().SerializeToString() + request = build_direct_cfl_logs_payload() + if encoding == "json": + for resource_logs in request.resource_logs: + for scope_logs in resource_logs.scope_logs: + for record in scope_logs.log_records: + record.trace_id = b"" + record.span_id = b"" + payload = json_format.MessageToJson(request).encode("utf-8") + content_type = "application/json" + else: + payload = request.SerializeToString() + content_type = "application/x-protobuf" statuses = [] for direct in [False, True]: @@ -3172,7 +3333,8 @@ def test_in_opentelemetry_logs_direct_cfl_limit_parity(tmp_path, limit): logs_max_size=str(limit)) try: service.start() - response = service.send_raw_request("/v1/logs", payload) + response = service.send_raw_request( + "/v1/logs", payload, content_type=content_type) statuses.append(response.status_code) finally: service.stop() From 31ae77c695e936bfa293fb7f4677175d7ee034f9 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 29 Oct 2026 00:13:11 -0600 Subject: [PATCH 19/33] tests: internal: compare OTLP logs CFL and msgpack decoding The input decodes OTLP logs straight to CFL records. Check that the JSON and protobuf CFL decoders accept, reject and size every request like the msgpack decoders do, using the shared JSON cases, payloads that exercise the leniency of the JSON conversion and generated request corpora. Signed-off-by: Eduardo Silva --- tests/internal/data/opentelemetry/README.md | 15 +- .../data/opentelemetry/logs_parity.jsonl | 200 ++++++++ .../data/opentelemetry/logs_parity.pb | Bin 0 -> 37812 bytes .../data/opentelemetry/logs_parity_corpus.py | 245 +++++++++ tests/internal/opentelemetry.c | 465 ++++++++++++++++++ 5 files changed, 924 insertions(+), 1 deletion(-) create mode 100644 tests/internal/data/opentelemetry/logs_parity.jsonl create mode 100644 tests/internal/data/opentelemetry/logs_parity.pb create mode 100755 tests/internal/data/opentelemetry/logs_parity_corpus.py diff --git a/tests/internal/data/opentelemetry/README.md b/tests/internal/data/opentelemetry/README.md index 656a34f073f..fa3005e7123 100644 --- a/tests/internal/data/opentelemetry/README.md +++ b/tests/internal/data/opentelemetry/README.md @@ -163,4 +163,17 @@ The test framework automatically detects whether a test case uses the legacy or - If `expected` contains a `groups` field, it uses the extended format - Otherwise, it falls back to the legacy format -This ensures all existing test cases continue to work without modification. \ No newline at end of file +This ensures all existing test cases continue to work without modification. +## CFL Parity Corpora + +`logs_parity.jsonl` (one OTLP/JSON request per line) and `logs_parity.pb` +(OTLP protobuf requests, each prefixed by its little-endian 32-bit length) are +random, often invalid, logs requests. The tests decode every request with both +the CFL and the msgpack decoders and expect the same records, the same +rejections and the same `logs_max_size` boundary. + +Regenerate them with: + +```shell +tests/integration/.venv/bin/python tests/internal/data/opentelemetry/logs_parity_corpus.py +``` diff --git a/tests/internal/data/opentelemetry/logs_parity.jsonl b/tests/internal/data/opentelemetry/logs_parity.jsonl new file mode 100644 index 00000000000..f8bbfe7adc6 --- /dev/null +++ b/tests/internal/data/opentelemetry/logs_parity.jsonl @@ -0,0 +1,200 @@ +{"resourceLogs": [{"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "1.25"}, "schemaUrl": "1.25", "scopeLogs": [true]}, {"resource": "abc", "scopeLogs": [{"schemaUrl": "aGVsbG8=", "logRecords": []}]}]} +{"resourceLogs": [{"resource": null, "scopeLogs": [{"schemaUrl": 1.5, "logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": 5, "droppedAttributesCount": "1.25", "flags": 1.5, "eventName": -5, "body": 1}, {"observedTimeUnixNano": "x", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": null, "flags": "aGVsbG8=", "body": 9223372036854775808}, {"severityText": 9223372036854775808, "droppedAttributesCount": -0.0, "eventName": "aGVsbG8="}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": 9223372036854775808, "scopeLogs": [{"scope": {"name": 9223372036854775808, "version": "123", "attributes": ["123", {"key": "k", "value": null}, {"key": {"stringValue": "w"}, "value": {"kvlistValue": 9223372036854775808}}], "droppedAttributesCount": "1.25"}}, {"scope": {"name": false, "version": -5, "attributes": [], "droppedAttributesCount": "aGVsbG8="}, "logRecords": {}}]}, {"schemaUrl": -0.0, "scopeLogs": [{"scope": "abc", "logRecords": [{"severityNumber": "abc", "traceId": "0123456789abcdef0123456789abcdef", "body": {}}]}, {"logRecords": [{"timeUnixNano": 1.5, "severityNumber": -0.0, "body": 0}, {"timeUnixNano": "9x", "severityNumber": "1.25", "attributes": "a", "traceId": "zz", "spanId": "0123456789abcdef", "flags": false, "eventName": null}, {"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": 5, "attributes": "-9", "droppedAttributesCount": 9223372036854775808, "spanId": "0123456789abcdef", "flags": "123"}]}]}]} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "a"}, "scopeLogs": [{"scope": {"name": -0.0, "version": "123", "attributes": [{"value": {"intValue": 9223372036854775808}}, {"key": null, "value": {"boolValue": "a"}}], "droppedAttributesCount": -0.0}, "logRecords": {}}, {"logRecords": [{"severityText": 1, "attributes": {}, "body": 9223372036854775808}, {"severityText": 9223372036854775808, "attributes": [], "droppedAttributesCount": false, "flags": -0.0, "eventName": 9223372036854775808, "body": {}}, {"observedTimeUnixNano": "", "severityNumber": -5, "severityText": "aGVsbG8=", "droppedAttributesCount": 1, "traceId": 5, "spanId": "0123", "flags": "s", "body": "1.25"}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": {"name": 1.5, "version": "-9", "attributes": [], "droppedAttributesCount": "abc"}, "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": []}, {"resource": -0.0, "scopeLogs": [{"scope": {}, "logRecords": [{"severityNumber": "1.25", "attributes": {}, "spanId": "0123456789abcdef", "body": 9223372036854775808}, -5, {"timeUnixNano": "", "attributes": [{"key": "", "value": false}, {"key": 5}, {"key": {"intValue": 1}, "value": [{"str": "123"}]}], "traceId": 5, "spanId": "", "body": {"b": [{"a": []}, {"str": {"bytesValue": 1}}]}}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"timeUnixNano": "1640995200000000000", "attributes": "", "droppedAttributesCount": false, "traceId": "0123456789abcdef0123456789abcdef", "flags": 0}]}, {"logRecords": {}}]}, "abc"]} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"scope": {}, "logRecords": [{"severityNumber": "aGVsbG8=", "severityText": -0.0, "droppedAttributesCount": "123", "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123", "body": "123"}, {"severityNumber": "-9", "attributes": {}, "droppedAttributesCount": true, "traceId": "", "flags": "a", "eventName": 1.5, "body": null}]}, {"scope": {"name": -0.0, "version": "-9", "attributes": [{"value": {"bytesValue": "abc"}}, {"key": null, "value": {"kvlistValue": {"values": [{"key": "k"}]}}}], "droppedAttributesCount": -5}, "logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": "x", "severityNumber": "aGVsbG8=", "droppedAttributesCount": null, "traceId": "0123456789abcdef0123456789abcdef", "flags": 9223372036854775808, "eventName": false}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "x", "severityNumber": 1, "attributes": false, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "", "flags": false}]}, {"scope": "aGVsbG8=", "logRecords": [{"severityText": 9223372036854775808, "droppedAttributesCount": -5, "traceId": "", "body": {"boolValue": "-9"}}, {"timeUnixNano": -1, "severityText": 1.5, "droppedAttributesCount": true, "traceId": "zz"}]}]}]} +{"resourceLogs": [{"schemaUrl": "abc", "scopeLogs": [{"scope": {"name": 1, "version": 1, "attributes": [0], "droppedAttributesCount": "-9"}, "logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "", "severityText": "1.25", "flags": "-9", "eventName": 1.5}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": 9223372036854775808, "logRecords": [{"droppedAttributesCount": "abc", "traceId": "0123456789abcdef0123456789abcdef", "spanId": ""}]}, {"scope": {}, "schemaUrl": "", "logRecords": []}, {"scope": 1, "logRecords": {}}]}]} +{"resourceLogs": [{"scopeLogs": [1.5, {"scope": {"name": "-9", "version": 1.5, "attributes": [], "droppedAttributesCount": "aGVsbG8="}, "logRecords": [{"severityText": -0.0, "droppedAttributesCount": "-9", "spanId": "0123", "eventName": "a"}, {"severityText": 9223372036854775808, "attributes": {}, "traceId": "", "spanId": null, "eventName": -0.0, "body": {"b": {"doubleValue": "a"}, "c": {"boolValue": false}}}]}, {"logRecords": -0.0}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": true}]} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": "s", "logRecords": [{"observedTimeUnixNano": "x", "flags": 1.5, "eventName": "123"}, {"timeUnixNano": "1640995200000000000", "severityNumber": null, "droppedAttributesCount": 1.5, "flags": 1.5, "body": "1.25"}]}, {"scope": {"name": -5, "version": -0.0, "attributes": {}, "droppedAttributesCount": "1.25"}}, {"logRecords": [{"timeUnixNano": "99999999999999999999999", "severityNumber": "123", "severityText": "1.25", "attributes": {}, "flags": ""}, {"timeUnixNano": "99999999999999999999999", "severityText": -5, "traceId": "zz", "eventName": "abc"}, {"timeUnixNano": "", "severityText": "-9", "traceId": ""}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": -0.0}, "schemaUrl": 9223372036854775808, "scopeLogs": [{"logRecords": [{"timeUnixNano": "1640995200000000000", "attributes": [], "traceId": "0123456789abcdef0123456789abcdef", "spanId": null, "eventName": null}]}, {"logRecords": [{"timeUnixNano": "", "traceId": 5, "flags": "a", "eventName": "123", "body": {"": 0}}, {"timeUnixNano": -1, "spanId": "0123"}]}, {"scope": {}, "schemaUrl": "123", "logRecords": [{"timeUnixNano": 1.5, "severityNumber": 0, "spanId": "", "eventName": "abc"}, {"observedTimeUnixNano": 5, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123456789abcdef", "eventName": "-9", "body": {"bytesValue": true}}]}]}, {"scopeLogs": []}]} +{"resourceLogs": [{"schemaUrl": "s", "scopeLogs": [{"logRecords": [{"timeUnixNano": "9x", "severityText": 0, "attributes": {}, "spanId": "0123", "body": "a"}, {"timeUnixNano": "9x", "severityText": -5, "traceId": 5, "flags": "s", "eventName": 1.5}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": 5}]}]}, {"resource": {"attributes": [], "droppedAttributesCount": "a"}, "scopeLogs": [{"logRecords": [{"severityNumber": "123", "severityText": false, "traceId": "zz"}, {"timeUnixNano": 1640995200000000000, "severityNumber": 9223372036854775808, "severityText": true, "traceId": "", "flags": null, "eventName": null}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"logRecords": []}]}]} +{"resourceLogs": [{"resource": {"attributes": [{"key": {"intValue": 1}, "value": {"bytesValue": "s"}}, {"key": "", "value": {}}, {"key": null, "value": {"kvlistValue": true}}], "droppedAttributesCount": 1.5}, "schemaUrl": -5, "scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": "", "observedTimeUnixNano": "x", "severityNumber": "", "severityText": "123", "attributes": {}, "flags": true, "body": -0.0}]}, 0, {"scope": "1.25", "logRecords": [{"timeUnixNano": "9x", "severityText": -5, "spanId": "", "flags": "123", "eventName": 9223372036854775808}]}]}, {"resource": {"attributes": [{"key": {"stringValue": "w"}, "value": {"intValue": 1}}], "droppedAttributesCount": 1.5}, "scopeLogs": [{"logRecords": []}]}]} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": ""}, "scopeLogs": [{"scope": {}, "schemaUrl": "s", "logRecords": [{"severityText": 1.5, "attributes": {}, "traceId": "0123456789abcdef0123456789abcdef"}, {"observedTimeUnixNano": 5, "severityNumber": "123", "severityText": "123", "attributes": {}, "eventName": "abc"}]}, {"logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "x", "severityText": -0.0, "droppedAttributesCount": -0.0, "spanId": null, "flags": 0}, {"severityText": "a", "spanId": ""}, {"timeUnixNano": -1, "severityNumber": true, "severityText": true, "attributes": {}, "droppedAttributesCount": 1, "traceId": ""}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": {"name": "a", "version": "123", "attributes": [], "droppedAttributesCount": "-9"}, "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"timeUnixNano": 1.5, "severityNumber": 1.5, "droppedAttributesCount": 1, "spanId": null, "flags": "a", "body": 9223372036854775808}, {"observedTimeUnixNano": 5, "severityText": 1, "droppedAttributesCount": "s", "eventName": "-9", "body": {}}]}, {"scope": {}, "logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "severityNumber": "s", "droppedAttributesCount": "abc", "spanId": "0123", "body": {"int": 1}}, {"timeUnixNano": "9x", "severityNumber": "", "spanId": "0123", "flags": null, "body": {}}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": null}, "schemaUrl": "-9", "scopeLogs": [{"logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "", "severityNumber": "123", "severityText": "1.25", "droppedAttributesCount": "aGVsbG8=", "traceId": "0123456789abcdef0123456789abcdeg"}]}]}, {"resource": -5, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": {"intValue": 1}, "value": {"kvlistValue": 0}}, {"value": {"intValue": 0}}, {"key": null, "value": {"arrayValue": "-9"}}], "droppedAttributesCount": 1.5}, "schemaUrl": "a", "scopeLogs": []}, {"resource": {"attributes": [{"key": "k", "value": {"intValue": 1}}, 1.5], "droppedAttributesCount": -0.0}, "schemaUrl": false, "scopeLogs": []}]} +{"resourceLogs": [{"schemaUrl": "s", "scopeLogs": [{}, {"logRecords": [{"timeUnixNano": -1, "droppedAttributesCount": "a", "spanId": null}]}]}, {"scopeLogs": [{}, {"logRecords": [{"timeUnixNano": "9x", "severityText": false, "attributes": {}, "droppedAttributesCount": 1, "spanId": "0123", "flags": "a"}, {"timeUnixNano": -1, "severityNumber": "aGVsbG8=", "attributes": [{"key": "k2", "value": {"arrayValue": [9223372036854775808, -0.0, -5]}}], "droppedAttributesCount": "1.25", "spanId": "0123", "body": ""}, {"timeUnixNano": 1.5, "observedTimeUnixNano": "5", "severityNumber": "abc", "droppedAttributesCount": "aGVsbG8=", "traceId": "", "spanId": null, "flags": null}]}, {"logRecords": []}]}]} +{"resourceLogs": [{"schemaUrl": false, "scopeLogs": [{"scope": {}, "schemaUrl": true, "logRecords": [{"severityNumber": "s", "severityText": true, "droppedAttributesCount": false, "traceId": "0123456789abcdef0123456789abcdeg", "flags": 1.5, "body": false}, {"severityNumber": 1.5, "droppedAttributesCount": 9223372036854775808, "spanId": "0123456789abcdef", "body": true}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "5", "severityNumber": 1, "attributes": 1.5, "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "", "flags": 1, "body": false}]}, {"logRecords": {}}, {"logRecords": -5}]}, {"resource": {"attributes": [{"key": "", "value": {"int": "s"}}, {"key": 5, "value": {"intValue": "-9"}}, {}], "droppedAttributesCount": 0}, "scopeLogs": [{"schemaUrl": -5, "logRecords": []}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": 9223372036854775808, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": -5, "scopeLogs": [{"scope": 1, "logRecords": [{"timeUnixNano": "", "severityText": null}]}]}, {"schemaUrl": "123", "scopeLogs": [{"logRecords": [{"droppedAttributesCount": "a", "traceId": "0123456789abcdef0123456789abcdeg", "eventName": -0.0}, {"timeUnixNano": 1.5, "severityNumber": "aGVsbG8=", "traceId": 5}]}, {"logRecords": [{"observedTimeUnixNano": "", "traceId": "zz", "flags": 9223372036854775808, "eventName": "-9"}, {"timeUnixNano": "1640995200000000000", "severityNumber": "abc", "severityText": 1.5, "attributes": [], "droppedAttributesCount": "s", "spanId": "0123456789abcdef", "flags": "-9", "eventName": "s"}, {"observedTimeUnixNano": "x", "severityNumber": null, "severityText": -5, "attributes": {}, "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123", "eventName": "-9"}]}, {"logRecords": 0}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": []}, {"schemaUrl": "a", "scopeLogs": [{"schemaUrl": null, "logRecords": []}, {}]}]} +{"resourceLogs": [{"resource": "aGVsbG8=", "schemaUrl": "abc", "scopeLogs": [{"logRecords": [{"severityNumber": false, "severityText": "123", "droppedAttributesCount": "", "spanId": null, "eventName": "", "body": {"a": {}, "c": {"kvlistValue": {}}}}, {"droppedAttributesCount": "-9", "spanId": ""}, true]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": false}, "scopeLogs": [{"scope": {"name": true, "version": "123", "attributes": [], "droppedAttributesCount": 9223372036854775808}, "logRecords": []}, {"logRecords": []}, {"logRecords": [{"observedTimeUnixNano": 5, "severityNumber": true, "droppedAttributesCount": null, "flags": 0, "body": "aGVsbG8="}]}]}]} +{"resourceLogs": [{"resource": true, "schemaUrl": 1.5, "scopeLogs": true}]} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": 0, "logRecords": [{"observedTimeUnixNano": "", "severityText": null, "body": {"": 9223372036854775808}}]}, {"scope": {}, "logRecords": [{"timeUnixNano": "9x", "traceId": "0123456789abcdef0123456789abcdef", "spanId": "", "flags": "123", "body": {}}]}, {"schemaUrl": "a", "logRecords": [{"observedTimeUnixNano": "5", "severityText": null, "droppedAttributesCount": 0, "spanId": null, "eventName": ""}]}]}, {"schemaUrl": -0.0, "scopeLogs": [{"logRecords": []}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": -0.0, "schemaUrl": 0, "scopeLogs": [{"scope": {"name": 0, "version": "abc", "attributes": [], "droppedAttributesCount": false}, "schemaUrl": null, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": 5, "severityText": "123", "attributes": [], "droppedAttributesCount": "-9", "spanId": "0123456789abcdef"}, {"observedTimeUnixNano": "5", "severityNumber": "-9", "attributes": {}, "traceId": "0123456789abcdef0123456789abcdeg", "flags": 0}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"scope": "123", "logRecords": ["s", {"timeUnixNano": "", "observedTimeUnixNano": "5", "attributes": 1, "eventName": -0.0, "body": 1}, {"observedTimeUnixNano": "x", "severityNumber": false, "droppedAttributesCount": 1.5, "flags": "1.25", "eventName": "1.25"}]}, {"scope": "aGVsbG8=", "logRecords": [{"timeUnixNano": 1.5, "severityText": -0.0, "droppedAttributesCount": "a", "traceId": "", "flags": "aGVsbG8=", "eventName": "s"}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": 9223372036854775808, "logRecords": [{"observedTimeUnixNano": 5, "severityNumber": "aGVsbG8=", "attributes": "aGVsbG8=", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": null, "flags": "aGVsbG8="}]}]}, {"schemaUrl": 9223372036854775808, "scopeLogs": [{"scope": {"name": "", "version": -0.0, "attributes": [], "droppedAttributesCount": "1.25"}, "schemaUrl": null, "logRecords": {}}]}]} +{"resourceLogs": [{"schemaUrl": null, "scopeLogs": [{"scope": {"name": "s", "version": false, "attributes": [], "droppedAttributesCount": "abc"}, "logRecords": [{"timeUnixNano": "", "severityNumber": "abc", "severityText": -0.0, "traceId": "", "spanId": "", "flags": "", "eventName": "abc"}, {"timeUnixNano": 1640995200000000000, "severityText": true, "attributes": {}, "droppedAttributesCount": "s", "traceId": "0123456789abcdef0123456789abcdeg"}, {"timeUnixNano": "9x", "observedTimeUnixNano": "", "severityNumber": false, "severityText": "a", "attributes": {}, "traceId": "", "eventName": 1}]}]}, {"scopeLogs": [{"logRecords": []}, {"logRecords": [{"severityNumber": "abc", "severityText": 9223372036854775808, "droppedAttributesCount": 9223372036854775808}]}, {}]}]} +{"resourceLogs": [{"scopeLogs": []}, {"schemaUrl": -0.0, "scopeLogs": [{"scope": -0.0, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "severityNumber": -5, "droppedAttributesCount": true}]}, {"logRecords": 0}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": 9223372036854775808, "scopeLogs": [{"scope": "123"}, {"scope": "a", "logRecords": []}, {"scope": null, "logRecords": [{"severityText": "abc", "droppedAttributesCount": -0.0, "spanId": "0123456789abcdef", "body": "s"}]}]}]} +{"resourceLogs": [{"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": "abc", "scopeLogs": [{"scope": {"name": 1.5, "version": 9223372036854775808, "attributes": [{"key": null, "value": {"arrayValue": 9223372036854775808}}], "droppedAttributesCount": 9223372036854775808}, "logRecords": [{"severityText": "a", "attributes": {}, "droppedAttributesCount": "-9", "body": ""}, {"observedTimeUnixNano": "x", "severityNumber": 1, "severityText": 1.5, "attributes": [{"key": {"stringValue": "w"}}, {"key": "k2", "value": {"a": "aGVsbG8=", "c": ""}}, {"key": "k", "value": {"boolValue": 0}}], "droppedAttributesCount": "123", "flags": "1.25", "eventName": "1.25", "body": null}, {"droppedAttributesCount": "abc", "flags": "-9", "body": -5}]}, {"logRecords": "123"}]}, {"resource": -5, "scopeLogs": "aGVsbG8="}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": 1, "logRecords": []}, {"scope": {"name": -0.0, "version": false, "attributes": [], "droppedAttributesCount": "aGVsbG8="}, "logRecords": []}, {"logRecords": [{"observedTimeUnixNano": "5", "severityNumber": null, "severityText": true, "attributes": [{"key": "k", "value": {"boolValue": 1}}, {"key": {"stringValue": "w"}, "value": {"boolValue": 1.5}}, {"key": "", "value": {}}], "droppedAttributesCount": "aGVsbG8=", "eventName": "abc"}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "5", "severityText": -5, "eventName": "aGVsbG8="}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "5", "severityText": "", "droppedAttributesCount": -5, "traceId": "", "spanId": null, "flags": "", "body": [{"": -5}, {"x": ""}]}]}]}]} +{"resourceLogs": [{"resource": -0.0, "scopeLogs": [{"logRecords": {}}, {"scope": {}, "logRecords": [{"observedTimeUnixNano": 5, "severityNumber": 1, "severityText": 1, "attributes": {}, "flags": null, "body": null}, {"observedTimeUnixNano": "", "severityText": true, "flags": false, "eventName": "s"}]}]}, {"resource": 1.5, "schemaUrl": 1.5, "scopeLogs": [{"scope": {"name": "", "version": "123", "attributes": {}, "droppedAttributesCount": -0.0}, "logRecords": [{"timeUnixNano": 1.5, "severityText": "aGVsbG8=", "attributes": -5, "eventName": "s", "body": {}}, {"severityText": 9223372036854775808, "attributes": 1, "droppedAttributesCount": "1.25", "spanId": "0123", "flags": "abc"}]}, {"logRecords": []}, {"scope": {"name": 9223372036854775808, "version": "abc", "attributes": [], "droppedAttributesCount": "s"}, "logRecords": []}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"value": {"doubleValue": 1.5}}, {"key": "", "value": ["aGVsbG8=", {"stringValue": "-9"}]}, {"key": "k2", "value": {"doubleValue": -5}}], "droppedAttributesCount": -0.0}, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": 0, "scopeLogs": [{"schemaUrl": -5, "logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "x", "severityText": "-9", "attributes": "s", "droppedAttributesCount": true, "flags": -5, "eventName": -5}, {"observedTimeUnixNano": "x", "attributes": "s", "droppedAttributesCount": "123", "flags": true, "body": {"stringValue": "a"}}]}, {"scope": "", "logRecords": [{"severityText": true, "droppedAttributesCount": false}, {"observedTimeUnixNano": "", "attributes": "aGVsbG8=", "droppedAttributesCount": 9223372036854775808, "spanId": "", "flags": "123"}, {"observedTimeUnixNano": "", "attributes": 0, "droppedAttributesCount": "abc", "traceId": 5, "spanId": "", "flags": -5, "body": -5}]}]}]} +{"resourceLogs": [{"schemaUrl": 9223372036854775808, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"scope": {}, "logRecords": []}, {"scope": {}, "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"severityText": "", "attributes": {}, "droppedAttributesCount": "123"}, {"severityNumber": 0, "severityText": 1.5, "attributes": true, "droppedAttributesCount": 9223372036854775808, "traceId": 5, "flags": 1, "body": "1.25"}, {"timeUnixNano": "99999999999999999999999", "traceId": "", "flags": false, "body": [{"bytesValue": "1.25"}, "abc"]}]}, {"scope": {"name": 0, "version": "s", "attributes": ["abc", {"key": {"intValue": 1}}], "droppedAttributesCount": 1}, "schemaUrl": -5, "logRecords": [{"timeUnixNano": "", "severityNumber": 0, "severityText": "a", "droppedAttributesCount": 9223372036854775808, "spanId": null, "flags": "", "eventName": "-9"}]}]}, {"scopeLogs": [{"schemaUrl": -5, "logRecords": [{"severityNumber": "123", "droppedAttributesCount": -5, "traceId": "0123456789abcdef0123456789abcdef"}, {"timeUnixNano": 1.5, "observedTimeUnixNano": 5, "severityText": false, "attributes": [{}, {"key": null}, {"key": "k2", "value": {"doubleValue": "1.25"}}], "droppedAttributesCount": "", "traceId": "", "spanId": "", "eventName": null}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "aGVsbG8="}, "scopeLogs": [{}, {"schemaUrl": 0, "logRecords": [{"severityNumber": "aGVsbG8=", "traceId": 5, "flags": null}, {"observedTimeUnixNano": 5, "attributes": true, "droppedAttributesCount": null, "traceId": "", "flags": -5, "body": {"doubleValue": "a"}}, {"observedTimeUnixNano": "5", "severityNumber": null, "severityText": false, "attributes": {}, "droppedAttributesCount": "", "spanId": ""}]}, {"scope": 0, "schemaUrl": "abc", "logRecords": [{"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "x", "severityNumber": false, "attributes": {}, "traceId": "0123456789abcdef0123456789abcdef", "eventName": "1.25", "body": {}}, {"attributes": {}, "droppedAttributesCount": "1.25", "traceId": 5, "flags": 0, "eventName": ""}, {"timeUnixNano": -1, "attributes": [{"key": null, "value": {"doubleValue": "a"}}, {"key": "k2", "value": "aGVsbG8="}, {"key": 5, "value": "plain"}], "spanId": "", "flags": "s", "eventName": "-9"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": "123", "scopeLogs": [{"logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "spanId": "0123456789abcdef", "eventName": 9223372036854775808, "body": {"doubleValue": {"stringValue": "123"}}}, {"observedTimeUnixNano": "x", "severityText": "a", "attributes": [{"value": {"bytesValue": "s"}}, {}, {"key": "", "value": {"": null}}], "droppedAttributesCount": 0, "spanId": "0123456789abcdef", "flags": false, "body": "aGVsbG8="}, {"timeUnixNano": 1640995200000000000, "severityNumber": 1.5, "attributes": {}, "spanId": "0123", "flags": -0.0, "eventName": "aGVsbG8=", "body": "s"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": "1.25", "scopeLogs": "aGVsbG8="}]} +{"resourceLogs": [{"scopeLogs": [{"scope": -5, "schemaUrl": null, "logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": "5", "flags": 9223372036854775808, "eventName": "abc"}]}]}, {"scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "5", "severityNumber": "1.25", "attributes": {}, "traceId": 5, "spanId": "", "flags": 1.5, "body": 1.5}, {"observedTimeUnixNano": "", "flags": null, "eventName": "1.25"}, {"severityNumber": "s", "attributes": 9223372036854775808, "droppedAttributesCount": 9223372036854775808, "traceId": "0123456789abcdef0123456789abcdef", "flags": "", "eventName": "aGVsbG8="}]}]}, {"scopeLogs": [{"logRecords": [{"attributes": {}, "flags": 0, "body": "s"}, {"observedTimeUnixNano": "5", "severityNumber": "123"}]}, {"schemaUrl": true, "logRecords": [{"observedTimeUnixNano": 5, "severityNumber": -5, "traceId": "0123456789abcdef0123456789abcdeg", "flags": -5, "eventName": "123"}]}]}]} +{"resourceLogs": [{"resource": null, "scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": 5, "droppedAttributesCount": 0, "flags": "s", "eventName": "1.25", "body": -5}, {"timeUnixNano": "1640995200000000000", "attributes": {}, "traceId": "0123456789abcdef0123456789abcdef", "flags": "aGVsbG8=", "body": {"intValue": "abc"}}, {"attributes": {}, "droppedAttributesCount": "123", "body": {}}]}]}, {"scopeLogs": [{"scope": {"name": "abc", "version": "", "attributes": {}, "droppedAttributesCount": 0}, "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": null, "scopeLogs": [{"logRecords": {}}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"timeUnixNano": "1640995200000000000", "severityNumber": "aGVsbG8=", "severityText": 1.5, "droppedAttributesCount": 1, "traceId": 5}, {"timeUnixNano": -1, "observedTimeUnixNano": "5", "droppedAttributesCount": 1.5, "flags": "abc"}, {"observedTimeUnixNano": "x", "severityNumber": "123", "attributes": [], "body": {"doubleValue": "a"}}]}]}, {"resource": {"attributes": {}, "droppedAttributesCount": null}, "scopeLogs": [{"logRecords": {}}, {"scope": {}, "logRecords": []}, {}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": null, "schemaUrl": "aGVsbG8=", "logRecords": [{"observedTimeUnixNano": "", "severityText": "s", "attributes": [{"key": 5, "value": {"boolValue": ""}}], "droppedAttributesCount": 0, "traceId": 5, "spanId": "0123456789abcdef", "flags": 0, "body": 1}, {"attributes": "1.25", "traceId": 5}, {"attributes": [{"key": "k2", "value": {"doubleValue": "aGVsbG8="}}, {"key": null, "value": {"kvlistValue": {"a": {"arrayValue": {"values": [["aGVsbG8=", null], 9223372036854775808]}}, "b": 1}}}], "droppedAttributesCount": -0.0, "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef"}]}]}, {"resource": 1.5, "schemaUrl": "aGVsbG8=", "scopeLogs": [{"scope": {}, "schemaUrl": 1, "logRecords": []}, {"logRecords": []}, {"scope": true, "schemaUrl": "a", "logRecords": [{"observedTimeUnixNano": 5, "severityText": 0, "attributes": [{"key": {"stringValue": "w"}, "value": "plain"}], "droppedAttributesCount": -5, "traceId": "zz", "spanId": "", "flags": "s"}, {"timeUnixNano": 1.5, "observedTimeUnixNano": "", "attributes": [{"key": "k", "value": "plain"}], "traceId": "0123456789abcdef0123456789abcdeg", "flags": "1.25", "body": {"b": {"boolValue": 0}}}, {"observedTimeUnixNano": "", "spanId": null, "flags": "aGVsbG8=", "eventName": "s", "body": null}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": "-9", "scopeLogs": [{"scope": {}, "logRecords": [{"severityNumber": false, "traceId": "", "flags": "s"}]}]}, {"resource": "aGVsbG8=", "schemaUrl": "", "scopeLogs": [{"logRecords": [{"severityNumber": -0.0, "attributes": {}, "droppedAttributesCount": null, "spanId": "0123"}, {"observedTimeUnixNano": "5", "droppedAttributesCount": 1.5, "traceId": 5, "spanId": "", "body": true}]}, {"schemaUrl": null, "logRecords": [{"timeUnixNano": 1.5, "observedTimeUnixNano": "5", "severityNumber": false, "attributes": 9223372036854775808}, {"severityNumber": "aGVsbG8=", "attributes": {}, "spanId": null, "flags": false}]}]}]} +{"resourceLogs": [{}]} +{"resourceLogs": [{"scopeLogs": [{}]}, {"schemaUrl": "abc", "scopeLogs": [{"logRecords": [{"severityNumber": "123", "severityText": "123", "attributes": {}, "droppedAttributesCount": -0.0, "eventName": -0.0, "body": {"arrayValue": []}}]}, {"scope": {}, "logRecords": []}, {"logRecords": [{"observedTimeUnixNano": "x", "severityText": -5, "traceId": "0123456789abcdef0123456789abcdef", "spanId": ""}, {"severityText": "aGVsbG8=", "attributes": [], "spanId": null, "flags": "aGVsbG8="}, {"timeUnixNano": "9x", "observedTimeUnixNano": 5, "traceId": "", "eventName": null}]}]}]} +{"resourceLogs": [{"resource": -5, "schemaUrl": 0, "scopeLogs": []}]} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": "123"}, "scopeLogs": [{"logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": "", "severityNumber": "aGVsbG8=", "severityText": "aGVsbG8=", "droppedAttributesCount": "123", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef", "flags": false, "eventName": "123", "body": {"a": {"int": ""}}}, {"severityNumber": "abc", "severityText": -5, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123456789abcdef", "flags": "1.25", "eventName": "abc"}, {"observedTimeUnixNano": 5, "severityNumber": "-9", "severityText": false, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123"}]}, {"schemaUrl": 9223372036854775808, "logRecords": {}}]}]} +{"resourceLogs": [{"resource": 1.5, "scopeLogs": [1, {"scope": -0.0}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": "aGVsbG8=", "logRecords": [{"traceId": 5, "spanId": null, "flags": "abc", "eventName": "", "body": {}}]}, {}, {"scope": {}, "logRecords": []}]}, {"resource": 1, "scopeLogs": [{"logRecords": [{"timeUnixNano": 1640995200000000000, "traceId": "0123456789abcdef0123456789abcdeg", "eventName": "123"}, {"observedTimeUnixNano": "", "droppedAttributesCount": 1, "eventName": -0.0}]}, {"scope": 0, "logRecords": []}]}]} +{"resourceLogs": [{"resource": -0.0, "schemaUrl": 9223372036854775808, "scopeLogs": [{"scope": "-9", "logRecords": [{"severityNumber": "123", "flags": "123", "eventName": true, "body": "1.25"}]}]}, {"schemaUrl": "", "scopeLogs": [{"scope": "1.25", "logRecords": []}, {"scope": {"name": -5, "version": -5, "attributes": [{"key": "k", "value": {}}], "droppedAttributesCount": ""}, "schemaUrl": 1, "logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": 5, "severityText": true, "attributes": 9223372036854775808, "traceId": "", "eventName": null, "body": {"stringValue": "123"}}, {"droppedAttributesCount": "aGVsbG8=", "traceId": "zz", "flags": "", "body": "-9"}]}, {"logRecords": [""]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": "s", "scopeLogs": [{"schemaUrl": false, "logRecords": []}, {"logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": 5, "droppedAttributesCount": null, "traceId": "", "spanId": "0123456789abcdef", "eventName": ""}]}, {"schemaUrl": 9223372036854775808, "logRecords": "aGVsbG8="}]}]} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": "-9"}, "scopeLogs": null}, {"resource": {"attributes": [{"key": 5, "value": {"boolValue": "a"}}, {"key": null, "value": {"boolValue": 1.5}}, {"key": "k2"}], "droppedAttributesCount": null}, "scopeLogs": []}]} +{"resourceLogs": [{"schemaUrl": 9223372036854775808, "scopeLogs": [{"scope": {"name": false, "version": null, "attributes": [{"key": {"stringValue": "w"}, "value": {"boolValue": -0.0}}, {"key": {"intValue": 1}, "value": {"doubleValue": 0}}, {"key": {"intValue": 1}, "value": {"doubleValue": "abc"}}], "droppedAttributesCount": ""}}, {"schemaUrl": "a", "logRecords": [{"severityNumber": "aGVsbG8=", "severityText": "1.25", "spanId": null, "eventName": "-9"}]}]}]} +{"resourceLogs": [{"resource": "1.25", "schemaUrl": "123", "scopeLogs": [{"logRecords": [{"severityText": 1.5, "traceId": "0123456789abcdef0123456789abcdeg", "flags": "aGVsbG8=", "eventName": "123"}, {"observedTimeUnixNano": "x", "severityNumber": "123", "attributes": [{"key": "", "value": ""}, {"key": "k2", "value": {"kvlistValue": true}}], "droppedAttributesCount": "abc", "traceId": "0123456789abcdef0123456789abcdeg", "flags": 1, "eventName": "aGVsbG8=", "body": {}}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": "aGVsbG8="}, "scopeLogs": []}, {"scopeLogs": "s"}]} +{"resourceLogs": [{"schemaUrl": true, "scopeLogs": [{"logRecords": ""}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": 5, "value": {"intValue": "abc"}}, {"key": "", "value": [-5]}], "droppedAttributesCount": "s"}, "scopeLogs": [{"scope": {}, "schemaUrl": false, "logRecords": [{"traceId": "", "spanId": null}, {"severityNumber": true, "attributes": {}, "traceId": "0123456789abcdef0123456789abcdeg", "flags": 9223372036854775808}]}]}]} +{"resourceLogs": [{"resource": 1, "scopeLogs": [{"logRecords": []}, {"schemaUrl": -5, "logRecords": [{"observedTimeUnixNano": "", "attributes": {}, "droppedAttributesCount": "1.25", "traceId": 5, "body": {}}, {"severityText": "s", "attributes": {}, "droppedAttributesCount": null, "traceId": "", "spanId": "0123", "flags": "123", "body": {}}, {"observedTimeUnixNano": "", "traceId": 5}]}, {"scope": {}, "logRecords": [{"flags": "a", "eventName": "aGVsbG8="}, {"timeUnixNano": 1.5, "attributes": 0, "droppedAttributesCount": false, "body": 9223372036854775808}, {"timeUnixNano": 1.5, "droppedAttributesCount": "s", "flags": null, "eventName": -5}]}]}, {"scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": [{}, 0]}]} +{"resourceLogs": [{"resource": "s", "scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": []}, {"resource": {"attributes": {}, "droppedAttributesCount": "-9"}, "scopeLogs": [{"logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "spanId": "", "flags": false, "eventName": -0.0, "body": {"str": {"str": ""}}}, {"timeUnixNano": "9x", "severityNumber": null, "traceId": "0123456789abcdef0123456789abcdeg", "spanId": ""}]}]}]} +{"resourceLogs": [{"schemaUrl": "", "scopeLogs": [{"schemaUrl": "123", "logRecords": []}, {"scope": {}, "logRecords": [{"timeUnixNano": "9x", "severityNumber": -5, "attributes": "aGVsbG8=", "traceId": ""}]}, {"scope": 1, "logRecords": []}]}]} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": 0}, "scopeLogs": ["123", {"scope": false, "logRecords": [{"timeUnixNano": 1640995200000000000, "severityText": -0.0, "spanId": null, "body": 1.5}, {"observedTimeUnixNano": 5, "severityNumber": -0.0, "attributes": {}, "traceId": 5, "flags": 0, "eventName": 1, "body": {}}, {"severityText": "s", "traceId": "zz", "spanId": "0123"}]}, {"logRecords": {}}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": -0.0, "logRecords": [{"timeUnixNano": "99999999999999999999999", "severityNumber": "1.25", "severityText": 1.5, "attributes": "123", "spanId": "", "eventName": "123", "body": ""}]}]}]} +{"resourceLogs": [{"resource": "s", "schemaUrl": "123", "scopeLogs": [{"scope": 1.5, "logRecords": true}, {"scope": {}, "logRecords": [{"observedTimeUnixNano": 5, "spanId": null, "eventName": "123", "body": []}, {"observedTimeUnixNano": "", "attributes": [], "traceId": "zz", "flags": true}, {"observedTimeUnixNano": "5", "severityText": "abc", "attributes": 1, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "", "eventName": 9223372036854775808, "body": false}]}, {"scope": {"name": true, "version": "-9", "attributes": {}, "droppedAttributesCount": "-9"}}]}, {"scopeLogs": [{"schemaUrl": "1.25"}, {"scope": {"name": 1.5, "version": "", "attributes": {}, "droppedAttributesCount": "-9"}, "logRecords": [{"droppedAttributesCount": 1, "spanId": null, "eventName": "abc"}]}, {"scope": {}, "logRecords": [{"severityNumber": "123", "spanId": ""}, {"timeUnixNano": "9x", "severityNumber": "a", "attributes": 1.5, "droppedAttributesCount": 9223372036854775808, "spanId": null, "flags": 1.5, "eventName": null, "body": "aGVsbG8="}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": 5, "severityText": "-9", "droppedAttributesCount": "1.25", "flags": "1.25", "body": {"int": 1.5}}]}]}]} +{"resourceLogs": [{"schemaUrl": -5, "scopeLogs": [{"scope": "123", "logRecords": []}, {"logRecords": [{"attributes": "123", "droppedAttributesCount": "1.25", "spanId": "0123456789abcdef", "eventName": 9223372036854775808}, {"timeUnixNano": 1640995200000000000, "severityText": "1.25", "attributes": "123", "droppedAttributesCount": "-9", "spanId": "0123", "flags": "aGVsbG8=", "eventName": "", "body": {"stringValue": 0}}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": 5, "severityText": 1.5, "traceId": 5, "spanId": "", "flags": "abc"}]}, {"schemaUrl": 9223372036854775808}]}, {"resource": {"attributes": {}, "droppedAttributesCount": 9223372036854775808}, "scopeLogs": [{"schemaUrl": "aGVsbG8=", "logRecords": {}}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": null}, "scopeLogs": [{"scope": null, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "severityNumber": true, "severityText": "1.25", "attributes": 1.5, "spanId": "0123", "eventName": -0.0, "body": {"doubleValue": "abc"}}, {"timeUnixNano": 1640995200000000000, "attributes": [], "droppedAttributesCount": "a", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef", "flags": -0.0, "eventName": "1.25", "body": true}]}, {"logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "eventName": "aGVsbG8="}]}]}]} +{"resourceLogs": [{"resource": "a", "scopeLogs": [{"logRecords": []}, {"scope": {"name": 1.5, "version": false, "attributes": [{"key": null}, {"key": "k2", "value": {"int": {"stringValue": 1.5}}}, {"key": "k2", "value": "plain"}], "droppedAttributesCount": null}, "logRecords": [{"timeUnixNano": 1.5, "observedTimeUnixNano": "", "severityNumber": "1.25", "eventName": -0.0}, {"timeUnixNano": "9x", "severityNumber": 9223372036854775808, "severityText": "", "eventName": "a"}]}, {"scope": {}, "logRecords": [{"severityNumber": -5, "attributes": true, "droppedAttributesCount": "", "traceId": "0123456789abcdef0123456789abcdef"}, {"attributes": {}, "droppedAttributesCount": "1.25", "traceId": "zz", "spanId": "0123456789abcdef", "body": {}}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "", "severityNumber": 9223372036854775808, "body": {}}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{}]} +{"resourceLogs": [{"resource": "123", "schemaUrl": true, "scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": 1.5, "attributes": [{"key": null, "value": 0}, {"key": "", "value": {"stringValue": "a"}}], "traceId": "", "flags": "abc"}, {"timeUnixNano": -1, "severityNumber": 9223372036854775808, "severityText": "1.25", "droppedAttributesCount": -5, "traceId": "0123456789abcdef0123456789abcdeg", "flags": 1, "eventName": 1.5}, {"observedTimeUnixNano": "5", "severityNumber": -5, "attributes": true, "spanId": "0123", "eventName": 1}]}, {"logRecords": [{"timeUnixNano": "9x", "severityNumber": "1.25", "droppedAttributesCount": "1.25", "traceId": "zz", "spanId": "0123", "body": true}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": -0.0}, "schemaUrl": "aGVsbG8=", "scopeLogs": [{"scope": -5}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": "-9", "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": -0.0, "scopeLogs": [{"scope": "aGVsbG8=", "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [1, {"observedTimeUnixNano": "5", "eventName": null, "body": {"kvlistValue": 9223372036854775808}}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "", "severityNumber": "", "droppedAttributesCount": 1.5, "traceId": 5, "eventName": "s"}]}]}, {"schemaUrl": "a", "scopeLogs": [{"scope": {}, "schemaUrl": -0.0, "logRecords": []}, {"schemaUrl": true, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "x", "severityNumber": "1.25", "severityText": 1.5, "attributes": [{"key": 5, "value": [{"x": "a"}]}], "droppedAttributesCount": true, "traceId": "zz"}]}]}]} +{"resourceLogs": [{"scopeLogs": true}]} +{"resourceLogs": [{}, {"resource": {"attributes": {}, "droppedAttributesCount": true}, "scopeLogs": []}]} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": ""}, "scopeLogs": [{"logRecords": []}]}, {"resource": -5, "schemaUrl": false, "scopeLogs": [{"logRecords": [{"timeUnixNano": 1.5, "observedTimeUnixNano": "", "severityNumber": 1.5, "spanId": "", "flags": true, "body": ""}, {"timeUnixNano": "", "severityNumber": true, "droppedAttributesCount": 0, "spanId": "0123456789abcdef"}]}, {"scope": {"name": "a", "version": -0.0, "attributes": {}, "droppedAttributesCount": "abc"}, "schemaUrl": 0, "logRecords": []}]}]} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": 0}, "scopeLogs": [{"scope": {}, "logRecords": []}]}, {"scopeLogs": [{"scope": {"name": 9223372036854775808, "version": -0.0, "attributes": [], "droppedAttributesCount": null}, "schemaUrl": "", "logRecords": [{"timeUnixNano": "99999999999999999999999", "severityNumber": "1.25", "traceId": "0123456789abcdef0123456789abcdeg"}, {"observedTimeUnixNano": 5, "severityText": false, "attributes": [{"key": {"intValue": 1}, "value": ""}, {"key": null}, {"key": "k", "value": {"arrayValue": "aGVsbG8="}}], "traceId": "zz", "spanId": null}, {"attributes": "s", "flags": true, "eventName": "a"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": 5}], "droppedAttributesCount": 1}, "schemaUrl": "1.25", "scopeLogs": [{"scope": "aGVsbG8=", "logRecords": [{"timeUnixNano": "", "severityNumber": "", "severityText": 9223372036854775808, "attributes": [{"key": {"intValue": 1}, "value": 1.5}, {"key": null, "value": "plain"}], "eventName": "-9", "body": "aGVsbG8="}]}]}, {"resource": {"attributes": [{"key": 5}], "droppedAttributesCount": "a"}, "scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "x", "severityNumber": "-9", "severityText": -0.0, "droppedAttributesCount": 0, "flags": "123", "eventName": "-9"}, {"timeUnixNano": 1640995200000000000, "severityNumber": "-9", "severityText": 1.5, "attributes": "abc", "droppedAttributesCount": -0.0, "traceId": "0123456789abcdef0123456789abcdef", "spanId": null}, {"severityText": "a", "eventName": "", "body": true}]}, {"schemaUrl": "-9", "logRecords": [{"attributes": {}, "droppedAttributesCount": "abc", "eventName": "", "body": {"x": "abc"}}, {"severityNumber": true, "severityText": 9223372036854775808, "attributes": {}, "droppedAttributesCount": -5, "body": false}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "x", "severityText": false, "droppedAttributesCount": 9223372036854775808, "traceId": "zz", "flags": 9223372036854775808, "eventName": 9223372036854775808, "body": 0}]}, {"logRecords": [{"observedTimeUnixNano": "x", "severityNumber": 1.5, "severityText": "1.25", "droppedAttributesCount": "1.25", "traceId": "zz", "spanId": null, "eventName": "-9", "body": null}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": -0.0, "scopeLogs": [{"scope": false, "schemaUrl": "1.25", "logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": "", "attributes": [], "flags": "abc", "body": "123"}, {"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "x", "severityText": -0.0, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123", "flags": 9223372036854775808, "eventName": "-9"}]}]}, {"scopeLogs": [{"scope": {}, "schemaUrl": true, "logRecords": [{"observedTimeUnixNano": "5", "severityNumber": "s", "severityText": 1, "traceId": "0123456789abcdef0123456789abcdef", "flags": "1.25"}, {"severityNumber": 1}]}, {"schemaUrl": 1.5, "logRecords": [{"timeUnixNano": 1.5, "observedTimeUnixNano": 5, "attributes": [{"key": 5, "value": {"arrayValue": "abc"}}, {"value": {"bytesValue": 1.5}}], "flags": "1.25", "eventName": "a", "body": {}}, {"severityNumber": 1, "severityText": "123", "traceId": "", "flags": "-9", "body": ["aGVsbG8=", [{"": "123"}], {"doubleValue": "a"}]}]}]}]} +{"resourceLogs": [{"resource": {"attributes": [{"key": 5}], "droppedAttributesCount": "1.25"}, "scopeLogs": []}, {"resource": 1.5, "scopeLogs": [{"scope": {}, "schemaUrl": 1, "logRecords": []}, {"scope": "123", "logRecords": []}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": -0.0}, "scopeLogs": [{"scope": 0, "logRecords": -5}]}, {"resource": {"attributes": {}, "droppedAttributesCount": 1.5}, "schemaUrl": "s", "scopeLogs": [{"logRecords": [null]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": "", "scopeLogs": []}, {"resource": "1.25"}]} +{"resourceLogs": [{"scopeLogs": [{"scope": -5, "schemaUrl": false, "logRecords": [{"attributes": 0, "droppedAttributesCount": 0, "traceId": "zz", "body": 9223372036854775808}]}]}, {"schemaUrl": 1.5, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "1.25"}, "schemaUrl": "1.25", "scopeLogs": []}, {"resource": true, "scopeLogs": [{"scope": 1.5, "logRecords": []}, {"logRecords": [{"observedTimeUnixNano": 5, "severityText": "a", "traceId": 5, "spanId": "0123", "flags": "", "eventName": 1.5, "body": {"doubleValue": 0}}, {"timeUnixNano": "1640995200000000000", "attributes": "s", "traceId": 5, "eventName": "abc"}, {"timeUnixNano": "1640995200000000000", "severityNumber": -5, "spanId": "", "eventName": 1.5, "body": {"boolValue": "abc"}}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": "aGVsbG8=", "logRecords": [{"observedTimeUnixNano": 5, "droppedAttributesCount": 0, "eventName": true, "body": 9223372036854775808}]}, {"scope": {"name": -5, "version": true, "attributes": [{"key": null, "value": {"boolValue": "abc"}}, {"key": ""}, {"key": "", "value": {"bytesValue": "-9"}}], "droppedAttributesCount": 1}, "schemaUrl": "abc", "logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "", "traceId": "", "flags": 9223372036854775808}, {"timeUnixNano": "99999999999999999999999", "severityText": -5, "spanId": "0123"}, {"timeUnixNano": 1.5, "observedTimeUnixNano": "x", "severityText": 1.5, "traceId": "0123456789abcdef0123456789abcdeg", "eventName": null}]}, {"schemaUrl": -0.0, "logRecords": []}]}, {"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": "k2", "value": {"stringValue": "1.25"}}], "droppedAttributesCount": -0.0}, "schemaUrl": "aGVsbG8=", "scopeLogs": [{"scope": {"name": "s", "version": null, "attributes": [{"key": "k2"}], "droppedAttributesCount": "123"}, "logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "x", "severityText": "1.25", "droppedAttributesCount": null, "traceId": "0123456789abcdef0123456789abcdef", "spanId": null, "eventName": null}]}]}]} +{"resourceLogs": [{"schemaUrl": 1, "scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "5", "severityNumber": null, "severityText": "123", "droppedAttributesCount": true, "traceId": "zz", "spanId": null, "eventName": "abc"}, {"timeUnixNano": "9x", "attributes": "a", "droppedAttributesCount": "", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef"}]}]}, {"resource": 1, "scopeLogs": [{"logRecords": []}, {"schemaUrl": "abc", "logRecords": []}, {"logRecords": null}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"logRecords": 1.5}]}, {"scopeLogs": "aGVsbG8="}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": true, "scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": [{"scope": -5, "schemaUrl": "aGVsbG8=", "logRecords": [{"timeUnixNano": "99999999999999999999999", "severityNumber": "a", "severityText": null, "traceId": "0123456789abcdef0123456789abcdef", "body": 9223372036854775808}, {"spanId": "0123456789abcdef", "flags": -5, "body": 9223372036854775808}, {"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "severityText": null, "droppedAttributesCount": "aGVsbG8=", "spanId": null, "eventName": "1.25"}]}]}, {"scopeLogs": [{"logRecords": [-5]}, {"scope": "", "logRecords": [{"observedTimeUnixNano": "5", "severityNumber": true, "traceId": "zz"}]}, {"logRecords": false}]}]} +{"resourceLogs": [{"resource": true, "scopeLogs": [{}, {"scope": {}, "schemaUrl": "1.25", "logRecords": []}, {"scope": "a", "logRecords": []}]}, {"resource": true, "scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": "1640995200000000000", "severityText": "s", "attributes": 0, "traceId": "0123456789abcdef0123456789abcdef", "spanId": null, "flags": "a", "eventName": "abc", "body": {"int": 9223372036854775808}}, {"observedTimeUnixNano": "", "severityNumber": "aGVsbG8=", "severityText": "a", "droppedAttributesCount": "123", "traceId": "zz", "spanId": "", "flags": true}, {"timeUnixNano": -1, "droppedAttributesCount": "1.25", "traceId": 5, "spanId": "0123", "eventName": ""}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": []}, {"schemaUrl": null, "logRecords": []}, {"logRecords": [{"severityText": null, "droppedAttributesCount": 1.5, "traceId": 5, "spanId": "", "body": "abc"}]}]}, {"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"logRecords": [{"severityNumber": "", "droppedAttributesCount": "1.25", "eventName": -5, "body": ""}, {"timeUnixNano": "", "severityNumber": "aGVsbG8=", "severityText": 1.5, "traceId": "zz", "flags": 1.5, "eventName": 0}]}, {"scope": "a", "logRecords": []}]}]} +{"resourceLogs": [{"schemaUrl": 9223372036854775808, "scopeLogs": [{"scope": {"name": "", "version": "abc", "attributes": [], "droppedAttributesCount": "-9"}, "logRecords": []}]}, {"resource": true, "schemaUrl": 0, "scopeLogs": [{"scope": {}}, {"logRecords": {}}, {"scope": "", "schemaUrl": "abc", "logRecords": [{"timeUnixNano": "", "observedTimeUnixNano": "", "flags": 9223372036854775808, "body": true}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"logRecords": []}, -0.0]}]} +{"resourceLogs": [{"schemaUrl": -5, "scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": []}, {"resource": 1, "scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": [{}, {"logRecords": []}]}, {"scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "severityNumber": "123", "severityText": 1.5, "droppedAttributesCount": "-9", "traceId": "", "body": {}}]}]}]} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"scope": {"name": true, "version": "1.25", "attributes": {}, "droppedAttributesCount": "1.25"}, "logRecords": [{"severityText": "a", "droppedAttributesCount": "123", "traceId": "zz"}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "x", "droppedAttributesCount": null, "flags": 1.5, "body": {}}, {"timeUnixNano": 1640995200000000000, "severityText": "123", "attributes": "s", "droppedAttributesCount": "1.25", "spanId": "", "flags": 0}]}, {"scope": {"name": 0, "version": null, "attributes": [{"key": null, "value": {"boolValue": true}}, {"value": {"arrayValue": 1.5}}, {"key": "", "value": "abc"}], "droppedAttributesCount": "-9"}, "logRecords": []}]}]} +{"resourceLogs": [{"schemaUrl": "1.25", "scopeLogs": [{"logRecords": {}}]}, {"scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "5", "severityText": 9223372036854775808, "attributes": [{"value": {"boolValue": "s"}}], "flags": "-9", "body": {}}, {"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "x", "severityText": 9223372036854775808, "attributes": -0.0, "droppedAttributesCount": null, "spanId": "0123", "body": true}]}]}]} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "severityNumber": -5, "severityText": 1.5, "attributes": [], "traceId": "0123456789abcdef0123456789abcdeg", "flags": 1.5}]}, {"scope": {}, "logRecords": [{"observedTimeUnixNano": "x", "severityText": 1, "attributes": {}, "droppedAttributesCount": true, "flags": "abc", "eventName": "1.25"}]}, {"scope": {}, "schemaUrl": -5, "logRecords": [{"timeUnixNano": 1640995200000000000, "severityText": -5, "droppedAttributesCount": 1.5, "traceId": "0123456789abcdef0123456789abcdef", "flags": "123"}, {"severityNumber": "abc", "severityText": "", "traceId": "0123456789abcdef0123456789abcdeg", "flags": false, "body": 1}, {"timeUnixNano": 1.5, "observedTimeUnixNano": "5", "attributes": {}, "traceId": "", "spanId": "", "eventName": 1.5}]}]}, {"scopeLogs": [{"scope": {}, "logRecords": [{"flags": "a"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "123"}, "scopeLogs": [{"schemaUrl": "1.25", "logRecords": [{"timeUnixNano": 1640995200000000000, "severityText": false, "body": true}, {"observedTimeUnixNano": "5", "severityNumber": "a", "traceId": 5, "eventName": false}]}]}, {"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": null, "scopeLogs": []}, {"schemaUrl": "123", "scopeLogs": [{"scope": {}, "schemaUrl": null}, {"scope": {"name": "s", "version": "123", "attributes": [{"key": "k2", "value": {"intValue": 1}}, {"key": "k", "value": {}}], "droppedAttributesCount": "s"}, "logRecords": ["-9", {"severityNumber": "abc", "severityText": "123", "attributes": [{"key": "k2", "value": "plain"}, {"key": {"intValue": 1}, "value": {"boolValue": 1}}], "spanId": null, "flags": "1.25", "eventName": "aGVsbG8="}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": "k2", "value": {"boolValue": -5}}, {"key": "k", "value": {"": 1}}, {"key": {"stringValue": "w"}, "value": {}}], "droppedAttributesCount": "123"}, "scopeLogs": [{"scope": true, "schemaUrl": "-9", "logRecords": [{"severityNumber": "s", "severityText": "1.25", "traceId": "zz", "eventName": "abc", "body": {}}]}, {"scope": "-9", "logRecords": [{"severityNumber": -0.0, "severityText": "1.25", "attributes": "-9", "flags": 1.5, "eventName": "", "body": {"a": false, "b": {"a": 0, "c": [-5, {"b": false}, "-9"]}}}]}]}, {"resource": "1.25", "scopeLogs": [{"schemaUrl": -5, "logRecords": [{"timeUnixNano": "1640995200000000000", "severityNumber": "", "attributes": null, "droppedAttributesCount": false}, {"timeUnixNano": -1, "attributes": {}}, {"severityNumber": "abc", "droppedAttributesCount": "123", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef", "eventName": -0.0}]}, {"logRecords": [{"observedTimeUnixNano": 5, "severityNumber": 0, "flags": "a", "eventName": true}, {"severityNumber": "-9", "attributes": "1.25", "droppedAttributesCount": "s", "eventName": -0.0}]}]}]} +{"resourceLogs": [{"resource": {"attributes": [{"value": {"stringValue": "s"}}, {"key": {"stringValue": "w"}}], "droppedAttributesCount": 1.5}, "scopeLogs": 9223372036854775808}, {"resource": {"attributes": {}, "droppedAttributesCount": "aGVsbG8="}, "scopeLogs": [{"scope": {}, "logRecords": []}, {"logRecords": [{"timeUnixNano": "99999999999999999999999", "attributes": [{"key": "k", "value": {"stringValue": "-9"}}, {"key": null}, {"key": "k", "value": {}}], "droppedAttributesCount": "123", "traceId": "0123456789abcdef0123456789abcdef", "spanId": null, "flags": 0}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"severityNumber": null, "attributes": {}, "spanId": "0123456789abcdef", "body": "abc"}, {"timeUnixNano": "1640995200000000000", "severityText": 1.5, "attributes": {}, "body": {"intValue": 1.5}}, {"observedTimeUnixNano": "x", "severityNumber": "1.25", "attributes": {}, "traceId": "0123456789abcdef0123456789abcdeg", "eventName": 9223372036854775808, "body": {"str": "aGVsbG8="}}]}, {"scope": false, "schemaUrl": "abc", "logRecords": ""}, {"schemaUrl": "abc", "logRecords": [{"observedTimeUnixNano": "x", "severityText": null, "attributes": "abc", "flags": "123", "body": {}}]}]}, {"resource": "abc", "scopeLogs": [{"scope": {"name": "1.25", "version": 9223372036854775808, "attributes": {}, "droppedAttributesCount": 1.5}, "logRecords": [{"severityText": 1.5, "attributes": [{"key": null, "value": 9223372036854775808}, {"key": "k", "value": {"bytesValue": "a"}}, {"key": "", "value": [{"doubleValue": {"arrayValue": {}}}, "s", {"intValue": false}]}], "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123456789abcdef", "eventName": 9223372036854775808, "body": "s"}]}, {"scope": "123", "logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "severityNumber": "s", "spanId": "", "flags": "abc"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": null, "value": {"c": {"int": 0}, "b": -0.0}}, {"key": "", "value": {"bytesValue": -5}}], "droppedAttributesCount": "abc"}, "schemaUrl": "s", "scopeLogs": [{"logRecords": [{"timeUnixNano": 1.5, "severityText": true, "attributes": 1.5, "spanId": "0123", "flags": "abc", "body": {"arrayValue": {"values": [{"stringValue": "123"}, {"intValue": 9223372036854775808}]}}}]}, {"logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": 5, "severityText": "1.25", "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123456789abcdef", "flags": "a", "eventName": true}, {"severityNumber": "a", "severityText": true, "attributes": [{"key": {"stringValue": "w"}, "value": {"boolValue": "1.25"}}, {"key": "k", "value": {"doubleValue": "abc"}}], "spanId": ""}, {"severityNumber": -5}]}]}, {"resource": {"attributes": {}, "droppedAttributesCount": null}, "scopeLogs": [{"logRecords": [{"timeUnixNano": -1, "severityNumber": true, "severityText": "", "attributes": {}, "droppedAttributesCount": -0.0, "traceId": 5}]}, {"schemaUrl": "1.25", "logRecords": []}, {"logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": 5, "severityText": -5, "droppedAttributesCount": 9223372036854775808, "spanId": null, "flags": "a", "body": {}}, {"traceId": "", "flags": "", "eventName": null, "body": 1.5}]}]}]} +{"resourceLogs": [{"resource": {"attributes": [{"key": {"stringValue": "w"}}], "droppedAttributesCount": -5}, "schemaUrl": -0.0, "scopeLogs": [{"schemaUrl": "-9", "logRecords": [{"severityText": true, "attributes": {}, "spanId": "0123", "flags": true, "eventName": false}]}, {"scope": {"name": "s", "version": 1.5, "attributes": [{"key": {"stringValue": "w"}, "value": {"bytesValue": "a"}}], "droppedAttributesCount": "a"}, "schemaUrl": -5}]}, {"scopeLogs": []}]} +{"resourceLogs": []} diff --git a/tests/internal/data/opentelemetry/logs_parity.pb b/tests/internal/data/opentelemetry/logs_parity.pb new file mode 100644 index 0000000000000000000000000000000000000000..6a68955b10944b67e22c2560418ed30a6139d0a0 GIT binary patch literal 37812 zcmbV#d$3(sdEegWO1AFO>uAZ6&OLg+r6tL-wruPAO19;PBwLnk{J_Q!43=zT5WW~) z%*)2+<=S8Zq>u*%S|AK9Buz;n6G&i`q%UtnP!p}w@pibzxDdQ zwfDIhj*mz3(K%NweUB%)jcH#pXg&pS8U~jqx|y_2FvQYpUB9RdsJ-!NLnJyr+(Lv_Got z8zynJfAuHb;=w!9t8P}Q$xpfq&BB?92Tz}AF6Q{Vo1NuiXDTy+FHB!uw7_9f%`*Oe za&x%Hi|6g>d_#A+yVNY0=~X{5^1a#YHkw85g7^g%VwSt5X7S)B=~W-AG3Ar}IPLsD zEHJ+8D(8pwKa_O~$&q}H-f$(qTIQBu-c_3a+j>~;w|e&dpbb@gQx>THAgM^=NSK3*>-Z@o z-(Hrv)ldu5fJw?*oKGLyH<^W)%ccpt-GRg-G-?4CEkZGsZxiOGL}PdBE4S1w&YXVn z{InY77_8p*ynF{K?y~=Oe`h;vkDBZRi23=R{d8>)_a`Hr#?XZ6i9q5z^YG3!yMpp5 z5ix}Y6opx@6nBMN9u1@MLW8v}neHjVyc=d@Q_nWz&(hwIr`MknEp4)Rc~cUta^ZQR z7-z<9qwsxtf&DF5D|?w6gHm}2``5Mmt=fI1HZN;#F_$X}j4_%CN=)ktWrR-(B!8$m zZSJtQHMh119^~}J^Vv+8eRfZ?yW@C*;=x`=SCjO}h#`IPd=q{(Q#DmHNd=7_0yk0I z8ukZ)QC*R=@2ahhqhO3QoZSuGshSlWa!CW};Tv*h#0ZyVsgKe?=^-g5vv7%_$)p!c zHD_OGSITVjFYKT-uza((F_zwx@PNKpc`A^iMX>)4VwhK(oo2iG-Bh#2tgNQJNVOQAYNkyaDnHOqv<49KH?$N9!U;mTZY#>3W+!`-0o?vjh zLNGgH_eE+Z?)4QS<;6$P(f+tb!&5({yc^rzgS z22jO9mhDyf7jZu8o3&;s2NgJD_|KrS(>)JQ@@VF9yE*uVq2nybuOMVj`Tv{j%6uD< z1iLEJzdxxyUY~y(rsTatz9XyxZo=0E{+;RB>JRwt_iFP%N6Suf>c`x5hET?@IFbkK z7{>;HR3PY#T|&!0JrOPaM!5U6&6Et8mTne$ZPHxZWp^}G?WE@rP!VhGl7GU*06Q8{ zc=O+C`@M)yY$jSJiM=L&)}L(FS_;3q7|VuB$KS4+_onuBd5FkApqCa1ZjRgQ-L)R- zQvFxCm0j$AoDAtOou9tCN8QS^UdZ zwlS9w%@l^AUk@`})8rfyTfmo{GFS7O3!@+pOE;MY9}X!nT$+c{UTO1pDir!b&;Dlc zkxMh-`h)47{aBrhu_G0xI2<8gK?Oz+%~5`J*j=H_0v0=lP&NSQG{v||j0lW5h7HO( zZzM5ipD=rdyOjWPt|XMX2=1caA^Env$gYFPT(e91lgftLLFCKH zg4$*qeip#(QYRV%>M9&RXMU-+KUd?IAFkaq?j7S$4)jWDXeiQ0o2%eIK({T;rgH7@ zdcYuID#vi+BK#^$xogt54jjs_9yYqb%U^Yym?^XxH{9r{{|x z;a+!S_P7z;%hR$K<#90!$c+~eyFOn%%HEpl_qc?ASDWib___V0+88oxx4yj=Ss|F?`lzd9AKMF^N83(L~icq37p^8 zvwNT$I~}dQr$`gLdhq<@J%}Nq@dH}33H!;~y}xdrgLPNNA<1h7wP3}=Wc;h$17?4- z&rBi+jcI`Lx~yf&9T5zBXJpoJ4D0$vMaZTHo15Wi8~Z89MkeS0ZA~U~Yh{&rZo>Xe z^q$v2@Wo?0Iv4T6(1?h#sT>}5B#PJ5J)zZONI41CYhK1>+|=A?4)e|B&80BNyZLsT z+X71iGUEl>1+SuwwO?#7fwJcE&aSfifHF3?btWYugs9ny=_JUrI&}phi!U3z zma_lDp4l7&L&jl|rbutA`B(P;(d_vI>y=Rqx4O+{qg!v*xK%MTKMnXYkUR4T0Lnm{4foIPUq z=ZQxO(EV0s(5w#DD9Na~ASB)O)reR(P~6s`JOh|Bpd=1_mC;5 z55OOV^a>udH3{Oji04X53Do#L&MMe8$ zns8*!K5cyU@nDPdPE`|IQXt}WZN3P;;S$ukv)C^R=mQgPtA5~(Bmt&!*-(-XLHJfc zAIS52^Ry4 z-W#ZOhhrd6+yc{b9E95MC*}W|FV`v{NT~U_8r2aMk%DWTLa+S#VxfI3!c6!6vESdA zr7KXknYL^wU}aoc0xdU&7&`&${G&>xQO}u!V0pI)0r^va$}Y1@qC4}{{1W;4#o9nC z&@XV8xf2N&foNr3cX-eqKr}O7!lVrPg8|0`D^#Z|dkb{xl6Ygm4Jg_Vie>my9QufbWJ&p4*^Sd(j`~HlK`p|G}jR}im1ZhSR~Uq&BJ-ZQ3u8V zxOQuR;zhs+84al^-|;b{?ZCB^#L8R+98uXVDx4bB-Wdt2qO2H@!gl~55<}NYkltmK z4Q3F596?o-dMaRf`))y@4pJ}|JBHjzv&ZfBDWr97=rJ-bqSh(85MTz{7&--|9ts`z zQ}&G*etP0$#yesk;2U&{hDu^MX7FK7eW0&oqjbzioCN2 zP!uE|t4aRfD+I#ICy^<@uly@AdZ*j&$-)&c(Op2=XM!?iqN@=hwv!Ib_5q$<8|9bK zgSv>oLL~ZTb7VN9$bE6MH_j9arK)0*v@DWDVArIk-x{;frw-8$1f3P?|0%`m`!yV_ zW+1o?tGSw!VmV4Q3kqM^Pu1oj*4?K0{lVryGMV8lQG{~U_Z~Q z4N56!L4l6$Y_s)FPSxO-i;H)kUTgAr07+31yujiuUjnV|*LuF1b~l)EY2)MFvx43#lPlUT_3oO|;(Om0v-M({ zgg861q9Yh$%wA*_f^5zp6>7g!;|K6=&1<-V)9%&&-lEx%m)aa@W)iR#=gIoYS-TqG zOjwldmHwzpQQ*n0%K{elQ;Ret0TQ>3e{W%Begmz7jA=Jud30t@I!&1nXT&D43O!aP%^mAADg>4spbAZu5Xc*E=7j`32Gjv&t0_o8!pV%llqKLYJwkl8XTOHJSmsxM3azaPJGmEX zhiG7~c_ZR1k!hw=R;qs|w_vXEOQgn)Vadw8Q&=yc2wT0%z(gg1WF>R<5b8{Wh!jNx z(B+({oG+vXUkMA55W22LV7%?tYx6y{mR}oSn$BC@auu_&_2B^EcH8DFi!U2)BfruFWgK z!0pV|U-!%-t?*#nrW&)A-6)3r0a>A_N)bvpU*PXgB!hOp>bwncU}l->5K?g3I<28o0G6ThX=KwyFUhV}BU zma@b-;`ImQ&S!e|9ZJ#lEER|CJ$d1djSlMp*e0QE5&pGVDKnZ!w{#N;x_8r-4NYD> zlb~XZ05Rnlu}koPEf`x!P{QRlZiqc45dGX}sbl(%LKDa zqjJy;7O|MU+bq!l!XC(k!reuvLt=**fof zfa6y+E;t=meTuUB@d=^_BD;`$cwr)F*c-*tFkPKe*Hj8!KW+|N_7Jd?N|Uc)T!=(r zBgu$vo)svfWvM^hbKDkl89BgK(*K}!JMKjq$D1d%72V~}3Cn(I2qA71SCjs+990<~nxY{Qk9r17*d z)i)?hzZ2SZnf?Fe+Dbs_*sDl5q1K@k^wE@&&I%2BbO;%1TnJ3Vlt6G{Fhr|ZFfcoODErY~B}Uq-gTawqih#si zAp|>ZFi3B`Ii(fz#-BiB~3ZO@jJ0Z{mJF6Ysa9jsab_ig{^~^Y^zh&7} zP&%nfK>ew>eMuB$jt~*;9*6);^%&ouwXd}|WKJ-X-iotDTS4Wvwg#^)h(Pu-DJaao ze?Q}|Q%gTEVUg6SR0jtj0dyt;zFf+P9e_atlnW0;glW2kOaD2HmW)I5Jlzv(C3CIpHnXMFT9o`-OdERfdO7|! zcc33`WRVCv>n5;2-S@%uui*C48!Mj*@lxm%PD7G$vU!^#q|{)q@9nPYo}z zR{bIz(ICM<_9pfgWi@2UlC6k#L^-Zb$`tl7Un%y+7voPcnf?Og$o?V~|5kT>b1kGO zr&c_76mE_4a#cBdHpAww096XL*U>n8PYZ%U(8XtU;YhLrol`bmXI2ko_!rN|1~$iB z$)r8yT?Vt~TNFXrsX~b+EoTmkj4p8Eo?IK@3o@@^Vny+ud|?xl~z~a2+|h zk=ul1Wy+uXl?!V!7wk1rf39c_p*>TCoP7-mpaj)unf1s3U2C6%%lIeUfD6C3CTe*U;PO! z)SvaxA^DF=exbLxqL?Bybb2w7J$rZn2T6&t-EpR=jOv$EG^ua@)S z4CLC%NIgeR7nCduMP4U8$R7dz<<4_l=K2Y6>-TGOy6Yd&t{~(uC9})|xUFR-UA(l$q1|?B6W~~QAm$~=EdSexr83Q=6G)8Z|4M~#3D27QvGva5=#}_wIJWCRb zxWOZNQIg#K7W+F6sk%Z&`9-cW+yR+W<$>yam3g9>GvQ33E;aK8xd>+|YPeKr zWd{8BN_4OVy_J+EpcKAg^}J=X0?HLEa2Re18LB)ex}I|mm*%a@c8&8hv7)*%Q$eJF z>A1vdXhh<M}ttqhuxX6az}|; zxYHOOj0Wti<<{6e-BZUfi&5>AGsMC7*^Apm0S@T8Es+X2!>Gt43z_Ao_nH;MGMGie zXIG?s2X5r;h$&UMw#?}Vt_%VwK@Ka0O)-h~Z)=#moOM76eGKy1vv%`HQ$Ec@iXv3C z3%DuX9vvr7Fse?ZG@I^B{^|Xzqm2E9KZJ`crBd6cP%A(~Ku4uR=ti-}hMsb-eLn>J zI3W+?`4_a^dnbFb&L$&qio4JPM)KzKf#PM@(rt{~$f89Ld*yPW3wfG!1H0Sqq`PM9 zjnBF=B&0Ma%r#@D?2~B-6NBZ&MAGXZsrJR5#c2|GJ|X*pU)VoGYc?D_Xo&Ve`F`M@ zZk{r;B*1IYl(Ug_Lt@Ny*^GvUx*c2#Pn0X1y^D3kSnF)6OF$iSG4NBcx>9NMrnI;g zsfLh(xR=4?4?I49D5^5Mv5Lw-&IBZD2Hh?VK>}l`X)h*hWW{1dR3=ardIykP!8L4* zV^d2^0s~FwgqG4TdlRBxAumN$ZW!aBU77m+>rQq_=Jmav{jaJ;nGx6fysvyu-5}*N zM;6Sw*QXrs8TX){B9A;A7ilIj%gUKBZ7TqTkR~;5Zb8g^(%j&V`@(1NaAF=4!E)Py zpBsIl{HBmwV2v}5fhKxh#TlxT`p)nRJ=||DSWh|FOoQFurQH zC5um)e`U*8d}0617_LcOw^hbYnYpB2$nb_tu$Fcj24!u$D;79B%uR)zBo?|XWmCHk z_E`xHRA=XymjIdwu4p^{upxv2*Q`2=5bXkTtQW)d7iptE-m`D;2Y>8Ui}5H6qS1P( zrKjU6CDY(rNIOX+^j8I&q_>{KZUxhCR?>~OK&gFu_6UxRQAJ^gtpil5*iEc(fAU6I z6G;zr+d^bQGFglhEJK2&((C5e%+!{Udd}cblxtBRl}mYoEDUGfRpmyEM1Ex9#3rJJ z_BDYQ=nh7;t^aJf966UYDY9?7kz%r5cKokByoSwaeDvB6I1 zg`6np%7lx>8I92}hK1(fy^>ri%0Hs1B0wc0M2pXK3W_AKli zod^S{lv%`^as2~|37AXwume#A=%H?t7>vr8D~YpnKB+^6Z`Gw7c6#R8%+V4$AuLwp z_A#99Q&t#z5D^SCY%=m?y7xXx5)K^m`llvi<}5d7U@mCE@_ynpRwErAu$4*t;~V5+ zLlprL$K`RSo}vY~S)j9qaJ%(dOg8U=JxfE&9&4=&-g@+Vh&e0{OGbdU(QiU7x@IH) zBSd0H8kj}n^7-063>f+fcUK7jnOF%J6ip+tzXHyqm^RE7Y7E$FLo>s}GpTf;l_XU4 zvuDP>bf3}7Je@Y)7RB1OK$wodS(+v6^he@CEH34G+TAKr9ywqqZVns&SBF!uq)7OG5?Hwwl=zjc`{al;x1) zea5@9y)=+dHgz~T_FUMtEHBIlgXn*Rz<-ikF<5lrsgV>RZfQuelt~8b5pr@_ZWV?j zU7H8&{qA1Ft{gW0jEcFCl*uiLlr8d#M^AXR6aWB15pbTEgixXiryy-$kd+M^w2~u8 z&|q(ZnH_^~DKc)UYEUeG$mU;Dz1{z9-fZ7w&oKM zqrci!B0;oJF@;`5w4`0$`4i6Ic*@P=?5qrTloNjgh4Ua^p>6#22^`+xB|_UnEqj1H zg@yxN_4qgdNftpo7)m9!1lt9UsSeiY9qgBDcTID(xst4$c0f7snv~435`6((<=psT zg*14W8r$~6Tt;nu7NPM5SFO6!IFu;`DNq6UjN_Kd;M{S{SUlDjOW>I&YP>+!q}qPc z_6RJ%zARADr-2qStJ0?x`{2cy`ULys&$6qoaO=ZbIjNl9kb)#kR7D2UeLEUpsr1f-(&CpB#Lds_y!9-H{ zPDnG1t^nn7x_4B(GC!Zk2$dZ`&P(fJP26) zVx|*L``0P0;1e=L$Ujr-v3socdZ4of#z?Vk#1@O_+}%ZMVkO zqmdURza26_^WXf&D*4 za@GD3;AyntSq{1B6CFUEs*k;ZCKhqw?JiLeLcC8=oSpjIIz-{~q7MO7q7Nb@a~5Zv z%^8lammjuM_MyblvBg1V73I;b_+C8jhMr6YzAKA}yUE>%>4aN@_dJXbSs!<);Zag! z)$L$&q+PGelb4eh-|ZP2rYJRAUlT$zTx95n>gEIWn0LN|+~x6<<8BUMg-NYm_Yx#U zuW%q>iVM+s-`YJ0Rl$GWuLyaU0*DIda9&kqZ*<(yaGjEiL+&!Qs~1ASOl5HZdlLEr z%OUTA7%@0lh#f!+Uw?%^cqD{9P2$d~7JKZsi-I3=obs>$kJnZ1B|7a9rll+j*rg%n z1hVzcAA57e3Y;_dAh9uD=Pl0Hkh4$yd}0z~Tl8T-TP=l;UoheM8*Fy^`yR7;;q-hs zp+j69k3kqC>@&=cl~a^VusOyNdGuLQmsXPfyd3xvR*2VG%u7N^uoFK*nCsD3F=NP-r&`rdH^ zafNhq12o#9a}u4gZMowD)u`%@v2>q_V?Wg+IcDv*YsZ7Y=ShL&lWctQ8SJGPiotO!o54=;K{r(F^>7yTFm*;`XP6MOcLF-?mbF zZDNZnpeuhS_A+Yk>ZHA}gd;cc`Cbaj6+6NWXbyuYXLHekdT_Ct-?ERDv`0|@tBT?C zU+|lUfq@>xm1Q{N^pkvAi3J>hUnb&C0h@vOrBp<7OaTAhLEAtf90* zW(Pq=y6y=_ouv+kd3T|hPc1~QP3DUs+j&M*>V8<;S7%EbZT3lbgE{W5H#{3P*AqJ^ zP{B(A`jp{=!YCmN)=R1u8X~+d6U=KvN>+4sV~p1vJ+00*$+BU8pH&4NPLHB#rbB;_ z(xk1R+5QiSwtz^ODJc>+?w+xnGL4Vff!6Y3O94{S0u&)~p=So(SYnCq zbDRTSvcyIg)0Z+>Z=?*g~cec##?{X>UYt(+{ zK}WG104rBn>;^U>*!noN9p72v?&`jQ@aJdXEirMv85<9^^P*37u2^+tUe(tDc>eC*dkoA8`Px zG6XQ}_eHZvDP;dq*gigXqzH-bDucb2PlsZL1M-vEpcCGb>358z$y!eV5IeS5ib9<; zSD^*kHodFPU z!tMPz#vQ3Q7(Edcmi0z&vIX2a1xy5viTJg;&ego_CnwBsSPcs!=TMF&wr?u?8@2mf z?XaJMK%dHzw`oVuprX!;)N-X16^zK~=GC}V3{b1SHjk-5>IP+hN;AuhV2ZRA!@d+y zt_pPzSu^&_+z0nv86NdA!?iW=S$ZA)#JiCYF`o`0K=|`*FYk>|MEGc)h*W9W_@uJTk~lC*F*k zCqkY4hAhE2vmRtc%+kfJm@_wVz#4H5b5WTQ#H%xb(f5YO_HEL|mtbs$EH)tb$j#5f zZH^Ten%-b(I5Pt|FJCb#8li_*s!MsQfqxXL`b^SN#y3M@^sqq~VmhcT8p;Q1u9iob z=`Ai={0DXt<~npb`lJ+S;%&G5=$!dx&wc~m*#0e!sUg|mg1_YZa{qPhzEGP_)D50k zl^L1ycszZgkWs*vmk8QSICwGK147y^3Ir2c$|>xrmnlke5bE)OcYyk*vdB0doe{zZ zCr8djqyc}kY0+fNJCJ1Yu?L`O5bWg(}bg5JTDp! zX-aCSD&f|ASw#wuDOSTvzw z@@WYONn24hx+}gLpjUxxdYt!Sb#4{*x=;D7Qz^>}OFc#Xus9*1+jrj_WwK1zAAF)3{U1yi2fIa4R!Zri@Bz-K+4LVHMWt`z? zT2ZgJe(P;KBGCsOXBoBI zFdEgH!y|50BqpIinB1NaXEz17E8K?=T6Vx-4>CGZCnDoVxTY zJ@f4vPf{ka(Yp6*`ZNY*G0iX54o`(X(>!Qi#h>2Wykh(`cj*w4V3tA#u^PhyLQrE9 za*{=Rzk2X;e{w`Jy=gn3vLfe8*qYo{{nL_o}<8QIrn|21jByvV~mBwNn-U9*x8RvWE?Vq zC+;}rHgFKR5_3AZ&Y>C*Gn#}DcNXk*xz8x%%SPm%K|yeD38$YbrzdmSMhwOo@sX2` z#kfc`^?aegOu^D<6xqdLxnq-TrpoDWwLA24N8Tqx0rw_{Vmq?JIY8cJnLh6<9xN;D zBXm3L$_qO&5T_&0i#T2JiU>+zhzS9rpS`SH*&hZZNkoJo)qNDGd%(UlL%X$j@NO^@ zAyiTum(kcV!W49y{rpCu|5Brwk&o>G8D+-yyK!5=&!wgkZflT~El9vM1>5(Ms{t>y ziU1R-4$a-Ao@G-^jQijFo+q%lRQJl&WX!Rkb|R>rJ>9GD)HKMztXbzIIb10 zbsSsz7Sv)e={C5BC0+Q&wEa+66eLisK1RsN{koGY7a0e?Wg7 zIb$AI!BB;J;~&v@)OJ@`H}*;bCQ37z&L=R5G*hjiC%OqRsz?&yvtXyw7P*y>2-$KO z(U(5t!G@h+n`-mT3H!flqK4Vwy~>}kP@ebiLtv2Ii&@TA4+VTo;aLP`NyY$+!UnZ0 z2DuxNdG41j8-DtUdok z8AYbv$|xo#?oqynZ2K30WN5*t(DQ@aG4f==l`dl(0qhjs 3 or choice < 0.3: + return json_scalar() + if choice < 0.7: + key = random.choice(JSON_WRAPPERS) + if key == "arrayValue": + value = random.choice([ + [json_any_value(depth + 1) for _ in range(random.randint(0, 3))], + {"values": [json_any_value(depth + 1) for _ in range(2)]}, + {}, {"other": 1}, json_scalar()]) + elif key == "kvlistValue": + value = random.choice([ + {"values": json_key_values(depth + 1)}, {}, + {"a": json_any_value(depth + 1), "b": 1}, + json_key_values(depth + 1), json_scalar()]) + elif random.random() < 0.8: + value = json_scalar() + else: + value = json_any_value(depth + 1) + return {key: value} + if choice < 0.85: + return [json_any_value(depth + 1) for _ in range(random.randint(0, 3))] + return {random.choice(["a", "b", "c"]): json_any_value(depth + 1) + for _ in range(random.randint(0, 3))} + + +def json_key_values(depth=0): + entries = [] + for _ in range(random.randint(0, 3)): + entry = {} + if random.random() < 0.9: + entry["key"] = random.choice(["k", "k2", "", None, 5, {"stringValue": "w"}, + {"intValue": 1}]) + if random.random() < 0.85: + if random.random() < 0.2: + entry["value"] = random.choice([{}, json_any_value(depth + 1), "plain"]) + elif random.random() < 0.3: + entry["value"] = json_any_value(depth + 1) + else: + entry["value"] = {random.choice(JSON_WRAPPERS[:7]): json_scalar()} + entries.append(entry if random.random() < 0.95 else json_scalar()) + return entries + + +def json_log_record(): + fields = [ + ("timeUnixNano", lambda: random.choice(["1640995200000000000", "", 1640995200000000000, + -1, 1.5, "9x", "99999999999999999999999"])), + ("observedTimeUnixNano", lambda: random.choice(["5", "x", 5, ""])), + ("severityNumber", json_scalar), + ("severityText", json_scalar), + ("attributes", lambda: random.choice([json_key_values(), {}, json_scalar()])), + ("droppedAttributesCount", json_scalar), + ("traceId", lambda: random.choice(["", "0123456789abcdef0123456789abcdef", "zz", 5, + "0123456789abcdef0123456789abcdeg"])), + ("spanId", lambda: random.choice(["", "0123456789abcdef", "0123", None])), + ("flags", json_scalar), + ("eventName", json_scalar), + ("body", lambda: random.choice([json_any_value(), json_any_value(), {}, json_scalar()])), + ] + record = {} + for name, generate in fields: + if random.random() < 0.45: + record[name] = generate() + return record if random.random() < 0.97 else json_scalar() + + +def json_scope_logs(): + scope_logs = {} + if random.random() < 0.5: + scope_logs["scope"] = random.choice([ + {"name": json_scalar(), "version": json_scalar(), + "attributes": random.choice([json_key_values(), {}, []]), + "droppedAttributesCount": json_scalar()}, + {}, json_scalar()]) + if random.random() < 0.3: + scope_logs["schemaUrl"] = json_scalar() + if random.random() < 0.95: + if random.random() < 0.1: + scope_logs["logRecords"] = random.choice([{}, json_scalar()]) + else: + scope_logs["logRecords"] = [json_log_record() for _ in range(random.randint(0, 3))] + return scope_logs if random.random() < 0.97 else json_scalar() + + +def json_resource_logs(): + resource_logs = {} + if random.random() < 0.5: + resource_logs["resource"] = random.choice([ + {"attributes": random.choice([json_key_values(), {}]), + "droppedAttributesCount": json_scalar()}, + json_scalar()]) + if random.random() < 0.3: + resource_logs["schemaUrl"] = json_scalar() + if random.random() < 0.97: + if random.random() < 0.95: + resource_logs["scopeLogs"] = [json_scope_logs() for _ in range(random.randint(0, 3))] + else: + resource_logs["scopeLogs"] = json_scalar() + return resource_logs if random.random() < 0.97 else json_scalar() + + +def json_request(): + return {"resourceLogs": [json_resource_logs() for _ in range(random.randint(0, 2))]} + + +def pb_any_value(value, depth=0): + choice = random.random() + if choice < 0.08: + return + if depth > 3 or choice < 0.55: + field = random.choice(["string_value", "bool_value", "int_value", "double_value", + "bytes_value"]) + setattr(value, field, { + "string_value": random.choice(["", "s", "x" * 40]), + "bool_value": random.choice([True, False]), + "int_value": random.choice([0, 1, -1, 2**62, -(2**63)]), + "double_value": random.choice([0.0, -1.5, 1e300]), + "bytes_value": random.choice([b"", b"\x00\xff", b"abc"]), + }[field]) + return + if choice < 0.78: + for _ in range(random.randint(0, 3)): + pb_any_value(value.array_value.values.add(), depth + 1) + if not value.array_value.values and random.random() < 0.5: + value.array_value.SetInParent() + return + pb_key_values(value.kvlist_value.values, depth + 1) + if not value.kvlist_value.values: + value.kvlist_value.SetInParent() + + +def pb_key_values(entries, depth=0): + for _ in range(random.randint(0, 3)): + entry = entries.add(key=random.choice(["k", "", "key.name", "k"])) + if random.random() < 0.9: + pb_any_value(entry.value, depth) + + +def pb_log_record(record): + if random.random() < 0.8: + record.time_unix_nano = random.choice([1640995200000000000, 1, 0]) + if random.random() < 0.5: + record.observed_time_unix_nano = random.choice([1640995200000000001, 0]) + if random.random() < 0.5: + record.severity_number = random.choice([0, 9, 24, 25]) + if random.random() < 0.4: + record.severity_text = random.choice(["", "INFO"]) + if random.random() < 0.7: + pb_any_value(record.body) + if random.random() < 0.5: + pb_key_values(record.attributes) + if random.random() < 0.3: + record.dropped_attributes_count = random.randint(0, 3) + if random.random() < 0.3: + record.flags = random.choice([0, 1, 257]) + if random.random() < 0.3: + record.trace_id = random.choice([b"", bytes(16), bytes(range(16)), b"\x01" * 8]) + if random.random() < 0.3: + record.span_id = random.choice([b"", bytes(8), bytes(range(8)), b"\x01" * 4]) + + +def pb_request(): + request = ExportLogsServiceRequest() + for _ in range(random.randint(0, 2)): + resource_logs = request.resource_logs.add() + if random.random() < 0.6: + pb_key_values(resource_logs.resource.attributes) + if random.random() < 0.3: + resource_logs.resource.dropped_attributes_count = random.randint(0, 5) + if random.random() < 0.2: + resource_logs.resource.SetInParent() + if random.random() < 0.3: + resource_logs.schema_url = random.choice(["", "https://r"]) + for _ in range(random.randint(0, 3)): + scope_logs = resource_logs.scope_logs.add() + if random.random() < 0.6: + scope = scope_logs.scope + if random.random() < 0.5: + scope.name = random.choice(["", "n"]) + if random.random() < 0.5: + scope.version = random.choice(["", "1"]) + if random.random() < 0.5: + pb_key_values(scope.attributes) + if random.random() < 0.3: + scope.dropped_attributes_count = random.randint(0, 3) + scope.SetInParent() + if random.random() < 0.3: + scope_logs.schema_url = random.choice(["", "https://s"]) + for _ in range(random.randint(0, 3)): + pb_log_record(scope_logs.log_records.add()) + return request.SerializeToString() + + +def main(): + count = int(sys.argv[1]) if len(sys.argv) > 1 else 200 + random.seed(int(sys.argv[2]) if len(sys.argv) > 2 else 7) + directory = os.path.dirname(os.path.abspath(__file__)) + + with open(os.path.join(directory, "logs_parity.jsonl"), "w") as output: + for _ in range(count): + output.write(json.dumps(json_request()) + "\n") + + with open(os.path.join(directory, "logs_parity.pb"), "wb") as output: + for _ in range(count): + request = pb_request() + output.write(struct.pack(" +#include #include #include #include @@ -609,6 +610,8 @@ void test_json_payload_get_wrapped_value() #define OTEL_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/logs.json" #define OTEL_TRACES_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/traces.json" #define OTEL_METRICS_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/metrics.json" +#define OTEL_JSON_PARITY_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/logs_parity.jsonl" +#define OTEL_PROTOBUF_PARITY_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/logs_parity.pb" static flb_sds_t test_normalize_json(const char *input) { @@ -1719,6 +1722,463 @@ void test_opentelemetry_cases() flb_free(cases_json); } +/* + * The input encodes OTLP JSON logs through the single-pass CFL decoder. Every + * shared case must produce the same records, or the same error, as the + * generic JSON to msgpack conversion. + */ +static int json_cfl_encode(struct flb_log_event_encoder *encoder, + const char *json, size_t len, size_t limit, int *error_status) +{ + int ret; + struct flb_mp_chunk_cobj *chunk; + struct flb_opentelemetry_logs_json_cfl *result; + struct flb_opentelemetry_logs_decode_options options; + + result = NULL; + options.metadata_key = "otlp"; + options.body_key = NULL; + options.output_limit = limit; + encoder->output_limit = limit; + + ret = flb_opentelemetry_logs_json_to_cfl(json, len, &options, &result, error_status); + if (ret != 0) { + flb_opentelemetry_logs_json_cfl_destroy(result); + return ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED ? ret : -1; + } + + chunk = flb_opentelemetry_logs_json_cfl_take_chunk(result); + if (chunk == NULL) { + flb_opentelemetry_logs_json_cfl_destroy(result); + return -1; + } + + chunk->log_encoder = encoder; + ret = flb_mp_chunk_cobj_encode_append(chunk); + chunk->log_encoder = NULL; + + flb_mp_chunk_cobj_destroy(chunk); + flb_opentelemetry_logs_json_cfl_destroy(result); + + return ret; +} + +/* + * Compare two encoded buffers event by event. Records without a timestamp + * get the current time on each path, any recent time is accepted for them. + */ +static int log_events_equal(char *buffer, size_t size, char *other, size_t other_size) +{ + int ret; + int other_ret; + time_t now; + struct flb_log_event event; + struct flb_log_event other_event; + struct flb_log_event_decoder decoder; + struct flb_log_event_decoder other_decoder; + + now = time(NULL); + flb_log_event_decoder_init(&decoder, buffer, size); + flb_log_event_decoder_init(&other_decoder, other, other_size); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + flb_log_event_decoder_read_groups(&other_decoder, FLB_TRUE); + + while (1) { + ret = flb_log_event_decoder_next(&decoder, &event); + other_ret = flb_log_event_decoder_next(&other_decoder, &other_event); + if (ret != FLB_EVENT_DECODER_SUCCESS || other_ret != FLB_EVENT_DECODER_SUCCESS) { + break; + } + if (flb_time_equal(&event.timestamp, &other_event.timestamp) == FLB_FALSE && + (event.timestamp.tm.tv_sec < now - 3600 || + other_event.timestamp.tm.tv_sec < now - 3600)) { + break; + } + if (msgpack_object_equal(*event.metadata, *other_event.metadata) == false || + msgpack_object_equal(*event.body, *other_event.body) == false) { + break; + } + } + + flb_log_event_decoder_destroy(&decoder); + flb_log_event_decoder_destroy(&other_decoder); + + return ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && + other_ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA; +} + +/* + * Decode one payload through both paths and compare the outcome. A payload + * the CFL decoder reports as unsupported is decoded by the msgpack path in + * the input, so it is equal by construction. + */ +static void json_cfl_check_parity(const char *name, const char *json, size_t limit) +{ + int ret; + int cfl_ret; + int error_status; + int cfl_error_status; + struct flb_log_event_encoder enc; + struct flb_log_event_encoder cfl_enc; + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + flb_log_event_encoder_init(&cfl_enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + enc.output_limit = limit; + + error_status = 0; + cfl_error_status = 0; + ret = flb_opentelemetry_logs_json_to_msgpack(&enc, json, strlen(json), NULL, + &error_status); + cfl_ret = json_cfl_encode(&cfl_enc, json, strlen(json), limit, &cfl_error_status); + + if (cfl_ret != FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED) { + TEST_CHECK_((ret == 0) == (cfl_ret == 0), "%s (limit %zu): result %i vs %i", + name, limit, ret, cfl_ret); + if (ret == 0 && cfl_ret == 0) { + TEST_CHECK_(log_events_equal(enc.output_buffer, enc.output_length, + cfl_enc.output_buffer, cfl_enc.output_length), + "%s (limit %zu): records differ", name, limit); + } + else if (limit == SIZE_MAX) { + TEST_CHECK_(error_status == cfl_error_status, "%s: error %i vs %i", + name, error_status, cfl_error_status); + } + } + + flb_log_event_encoder_destroy(&enc); + flb_log_event_encoder_destroy(&cfl_enc); +} + +/* Payloads that exercise the leniency of the established conversion */ +void test_opentelemetry_json_cfl_parity_values() +{ + size_t i; + char json[2048]; + static const char *bodies[] = { + "{\"stringValue\":\"text\"}", + "{\"stringValue\":null}", + "{\"str\":\"prefix of stringValue\"}", + "{\"\":\"empty wrapper key\"}", + "{\"intValue\":\"123\"}", + "{\"intValue\":\"-42abc\"}", + "{\"intValue\":\"not a number\"}", + "{\"intValue\":7}", + "{\"intValue\":-7}", + "{\"intValue\":1.5}", + "{\"doubleValue\":\"1.5\"}", + "{\"doubleValue\":2.25}", + "{\"doubleValue\":3}", + "{\"boolValue\":\"true\"}", + "{\"bytesValue\":\"aGVsbG8=\"}", + "{\"bytesValue\":\"aGVsbG8\"}", + "{\"bytesValue\":\"a\"}", + "{\"arrayValue\":{}}", + "{\"arrayValue\":{\"values\":[{\"intValue\":\"1\"},null,[1,2]]}}", + "{\"arrayValue\":{\"other\":[]}}", + "{\"arrayValue\":[1,{\"stringValue\":\"x\"}]}", + "{\"kvlistValue\":{}}", + "{\"kvlistValue\":{\"values\":[]}}", + "{\"kvlistValue\":{\"a\":1,\"b\":2}}", + "{\"kvlistValue\":{\"values\":[{\"key\":\"k\"}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":\"k\",\"value\":{}}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":\"k\",\"value\":\"plain\"}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":{\"stringValue\":\"w\"}," + "\"value\":{\"intValue\":\"1\"}}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":null,\"value\":{\"boolValue\":true}}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":5,\"value\":{\"boolValue\":true}}]}}", + "{\"kvlistValue\":{\"values\":[{\"value\":{\"boolValue\":true}}]}}", + "{\"kvlistValue\":{\"values\":[\"not a map\"]}}", + "{\"kvlistValue\":[]}", + "{}", + "{\"plain\":{\"nested\":[1,null,true]},\"other\":\"x\"}", + "{\"custom\":{\"child\":[1,true,null]}}", + "{\"unknownValue\":1}", + "{\"stringValue\":\"a\",\"extra\":1}", + }; + static const char *records[] = { + "{\"timeUnixNano\":\"\"}", + "{\"timeUnixNano\":\"12345678901234567890123456789012345\"}", + "{\"timeUnixNano\":1722904465173450100}", + "{\"timeUnixNano\":-1}", + "{\"timeUnixNano\":1.5}", + "{\"timeUnixNano\":\"12a\"}", + "{\"observedTimeUnixNano\":\"77\"}", + "{\"timeUnixNano\":\"1\",\"observedTimeUnixNano\":5}", + "{\"timeUnixNano\":\"1\",\"observedTimeUnixNano\":\"x\"}", + "{\"timeUnixNano\":\"1\",\"observedTimeUnixNano\":\"18446744073709551615\"}", + "{\"timeUnixNano\":\"1\",\"severityNumber\":{\"a\":[1,null]}}", + "{\"timeUnixNano\":\"1\",\"severityNumber\":null,\"severityText\":5}", + "{\"timeUnixNano\":\"1\",\"flags\":\"257\"}", + "{\"timeUnixNano\":\"1\",\"traceFlags\":3}", + "{\"timeUnixNano\":\"1\",\"flags\":-3}", + "{\"timeUnixNano\":\"1\",\"flags\":\"3x\"}", + "{\"timeUnixNano\":\"1\",\"droppedAttributesCount\":-2}", + "{\"timeUnixNano\":\"1\",\"droppedAttributesCount\":\"2\"}", + "{\"timeUnixNano\":\"1\",\"traceId\":\"\",\"spanId\":\"\"}", + "{\"timeUnixNano\":\"1\",\"traceId\":5}", + "{\"timeUnixNano\":\"1\",\"attributes\":{}}", + "{\"timeUnixNano\":\"1\",\"attributes\":[{\"key\":\"a\",\"value\":{\"x\":1}}]}", + "{\"timeUnixNano\":\"1\",\"eventName\":\"e\",\"body\":\"s\"}", + "{\"TIMEUNIXNANO\":\"9\",\"BODY\":{\"STRINGVALUE\":\"upper\"}}", + }; + static const char *groups[] = { + "{\"resource\":{\"attributes\":[],\"droppedAttributesCount\":{\"x\":null}}," + "\"schemaUrl\":5,\"scopeLogs\":[{\"schemaUrl\":\"s\",%s}]}", + "{\"resource\":5,\"scopeLogs\":[{\"scope\":5,%s}]}", + "{\"scopeLogs\":[{\"scope\":{\"name\":5,\"version\":\"v\"," + "\"droppedAttributesCount\":null,\"attributes\":[]},%s}]}", + "{\"scopeLogs\":[{\"scope\":{\"attributes\":{}},%s}]}", + "{\"scopeLogs\":[{\"scope\":{\"attributes\":[\"x\"]},%s}]}", + "{\"resource\":{\"attributes\":[1]},\"scopeLogs\":[{%s}]}", + "{\"scopeLogs\":[{\"logRecords\":[]},{\"scope\":{}},{%s}]}", + }; + char inner[1024]; + + for (i = 0; i < sizeof(bodies) / sizeof(bodies[0]); i++) { + snprintf(json, sizeof(json), + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[" + "{\"timeUnixNano\":\"1\",\"body\":%s," + "\"attributes\":[{\"key\":\"a\",\"value\":%s}]}]}]}]}", + bodies[i], bodies[i]); + json_cfl_check_parity(bodies[i], json, SIZE_MAX); + } + for (i = 0; i < sizeof(records) / sizeof(records[0]); i++) { + snprintf(json, sizeof(json), + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[%s]}]}]}", + records[i]); + json_cfl_check_parity(records[i], json, SIZE_MAX); + } + for (i = 0; i < sizeof(groups) / sizeof(groups[0]); i++) { + snprintf(inner, sizeof(inner), groups[i], + "\"logRecords\":[{\"timeUnixNano\":\"1\",\"body\":{\"stringValue\":\"g\"}}]"); + snprintf(json, sizeof(json), "{\"resourceLogs\":[%s]}", inner); + json_cfl_check_parity(groups[i], json, SIZE_MAX); + } +} + +/* Generated requests, see data/opentelemetry/logs_parity_corpus.py */ +void test_opentelemetry_json_cfl_parity_corpus() +{ + int ret; + int count; + char name[32]; + char *corpus; + char *line; + char *end; + size_t size; + size_t limit; + struct flb_log_event_encoder enc; + + corpus = mk_file_to_buffer(OTEL_JSON_PARITY_PATH); + TEST_CHECK(corpus != NULL); + if (corpus == NULL) { + return; + } + + count = 0; + for (line = corpus; *line != '\0'; line = end + 1) { + end = strchr(line, '\n'); + if (end == NULL) { + break; + } + *end = '\0'; + snprintf(name, sizeof(name), "request %d", ++count); + json_cfl_check_parity(name, line, SIZE_MAX); + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + ret = flb_opentelemetry_logs_json_to_msgpack(&enc, line, strlen(line), NULL, NULL); + size = enc.output_length; + flb_log_event_encoder_destroy(&enc); + if (ret == 0) { + for (limit = size > 16 ? size - 16 : 0; limit <= size + 16; limit++) { + json_cfl_check_parity(name, line, limit); + } + } + } + flb_free(corpus); + TEST_CHECK(count > 0); +} + +/* + * The protobuf CFL decoder must produce the records of the msgpack decoder + * and enforce logs_max_size at the same size. The input encodes CFL records + * after processing, the limit only applies to the decoder estimate. + */ +static void protobuf_cfl_check_parity(const char *name, const char *payload, size_t length, + size_t limit) +{ + int ret; + int cfl_ret; + size_t record_count; + struct flb_mp_chunk_cobj *chunk; + struct flb_opentelemetry_logs_protobuf_cfl *result; + struct flb_opentelemetry_logs_decode_options options; + struct flb_log_event_encoder enc; + struct flb_log_event_encoder cfl_enc; + + options.metadata_key = "otlp"; + options.body_key = NULL; + options.output_limit = limit; + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + flb_log_event_encoder_init(&cfl_enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + enc.output_limit = limit; + + ret = flb_opentelemetry_logs_protobuf_to_msgpack(&enc, payload, length, &options, + &record_count); + + result = NULL; + cfl_ret = flb_opentelemetry_logs_protobuf_to_cfl(payload, length, &options, &result); + if (cfl_ret == 0) { + chunk = flb_opentelemetry_logs_protobuf_cfl_take_chunk(result); + chunk->log_encoder = &cfl_enc; + cfl_ret = flb_mp_chunk_cobj_encode_append(chunk); + chunk->log_encoder = NULL; + flb_mp_chunk_cobj_destroy(chunk); + } + flb_opentelemetry_logs_protobuf_cfl_destroy(result); + + TEST_CHECK_((ret == 0) == (cfl_ret == 0), "%s (limit %zu): result %i vs %i", + name, limit, ret, cfl_ret); + if (ret == 0 && cfl_ret == 0) { + TEST_CHECK_(log_events_equal(enc.output_buffer, enc.output_length, + cfl_enc.output_buffer, cfl_enc.output_length), + "%s (limit %zu): records differ", name, limit); + } + + flb_log_event_encoder_destroy(&enc); + flb_log_event_encoder_destroy(&cfl_enc); +} + +void test_opentelemetry_protobuf_cfl_parity_corpus() +{ + int ret; + int count; + char name[32]; + char *payload; + size_t size; + size_t limit; + size_t record_count; + uint32_t length; + FILE *file; + struct flb_log_event_encoder enc; + struct flb_opentelemetry_logs_decode_options options; + + file = fopen(OTEL_PROTOBUF_PARITY_PATH, "rb"); + TEST_CHECK(file != NULL); + if (file == NULL) { + return; + } + + options.metadata_key = "otlp"; + options.body_key = NULL; + options.output_limit = SIZE_MAX; + count = 0; + + /* each request is prefixed by its little-endian 32-bit length */ + while (fread(&length, sizeof(length), 1, file) == 1) { + payload = flb_malloc(length + 1); + TEST_CHECK(payload != NULL); + if (payload == NULL || fread(payload, 1, length, file) != length) { + flb_free(payload); + break; + } + snprintf(name, sizeof(name), "request %d", ++count); + protobuf_cfl_check_parity(name, payload, length, SIZE_MAX); + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + ret = flb_opentelemetry_logs_protobuf_to_msgpack(&enc, payload, length, &options, + &record_count); + size = enc.output_length; + flb_log_event_encoder_destroy(&enc); + if (ret == 0) { + for (limit = size > 16 ? size - 16 : 0; limit <= size + 16; limit++) { + protobuf_cfl_check_parity(name, payload, length, limit); + } + } + flb_free(payload); + } + fclose(file); + TEST_CHECK(count > 0); +} + +void test_opentelemetry_cases_json_cfl_parity() +{ + int ret; + int type; + char case_name[128]; + char *cases_json; + char *tmp_buf; + char *input_json; + size_t tmp_size; + size_t size; + size_t limit; + size_t i; + size_t input_index; + msgpack_unpacked result; + msgpack_object *root; + msgpack_object *case_obj; + struct flb_log_event_encoder enc; + + cases_json = mk_file_to_buffer(OTEL_TEST_CASES_PATH); + TEST_CHECK(cases_json != NULL); + if (cases_json == NULL) { + return; + } + + ret = flb_pack_json(cases_json, strlen(cases_json), &tmp_buf, &tmp_size, &type, NULL); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_free(cases_json); + return; + } + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, tmp_buf, tmp_size, NULL); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + root = &result.data; + + for (i = 0; i < root->via.map.size; i++) { + case_obj = &root->via.map.ptr[i].val; + + ret = flb_otel_utils_find_map_entry_by_key(&case_obj->via.map, "input", 0, FLB_TRUE); + TEST_CHECK(ret >= 0); + if (ret < 0) { + continue; + } + input_index = ret; + input_json = flb_msgpack_to_json_str(1024, &case_obj->via.map.ptr[input_index].val, + FLB_TRUE); + TEST_CHECK(input_json != NULL); + if (input_json == NULL) { + continue; + } + + snprintf(case_name, sizeof(case_name), "%.*s", + (int) root->via.map.ptr[i].key.via.str.size, + root->via.map.ptr[i].key.via.str.ptr); + json_cfl_check_parity(case_name, input_json, SIZE_MAX); + + /* the logs_max_size boundary must be the same on both paths */ + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + ret = flb_opentelemetry_logs_json_to_msgpack(&enc, input_json, strlen(input_json), + NULL, NULL); + size = enc.output_length; + flb_log_event_encoder_destroy(&enc); + if (ret == 0) { + for (limit = size > 64 ? size - 64 : 0; limit <= size + 64; limit++) { + json_cfl_check_parity(case_name, input_json, limit); + } + } + flb_free(input_json); + } + + msgpack_unpacked_destroy(&result); + flb_free(tmp_buf); + flb_free(cases_json); +} + + void test_opentelemetry_traces_cases() { int ret; @@ -4063,6 +4523,11 @@ TEST_LIST = { { "find_map_entry_by_key", test_find_map_entry_by_key }, { "json_payload_get_wrapped_value", test_json_payload_get_wrapped_value }, { "opentelemetry_cases", test_opentelemetry_cases }, + { "opentelemetry_cases_json_cfl_parity", test_opentelemetry_cases_json_cfl_parity }, + { "opentelemetry_json_cfl_parity_values", test_opentelemetry_json_cfl_parity_values }, + { "opentelemetry_json_cfl_parity_corpus", test_opentelemetry_json_cfl_parity_corpus }, + { "opentelemetry_protobuf_cfl_parity_corpus", + test_opentelemetry_protobuf_cfl_parity_corpus }, { "opentelemetry_logs_otlp_json_roundtrip", test_opentelemetry_logs_otlp_json_roundtrip }, { "opentelemetry_logs_otlp_json_preserves_appended_resources", From 8423e9de65113e9f8fe78db8b54f2f8c417cfc10 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 23:05:05 -0600 Subject: [PATCH 20/33] downstream: allow adopting an accepted connection Add flb_downstream_conn_event_adopt() so a socket accepted on one event loop can be handed to another worker and driven through the regular downstream connection setup, including TLS session creation. The adopted socket is always consumed: it is closed on every failure path. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_downstream.h | 12 ++++ src/flb_downstream.c | 85 ++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/include/fluent-bit/flb_downstream.h b/include/fluent-bit/flb_downstream.h index 6bb446c056d..12c1a0b3542 100644 --- a/include/fluent-bit/flb_downstream.h +++ b/include/fluent-bit/flb_downstream.h @@ -98,6 +98,18 @@ int flb_downstream_conn_event_accept( void *accept_callback_data, flb_connection_event_callback event_callback, int mask); + +/* + * Same as flb_downstream_conn_event_accept() for a socket that was already + * accepted, for example by another worker. The socket is always consumed. + */ +int flb_downstream_conn_event_adopt( + struct flb_downstream *stream, + flb_sockfd_t fd, + flb_connection_accept_callback accept_callback, + void *accept_callback_data, + flb_connection_event_callback event_callback, + int mask); int flb_downstream_conn_event_register(struct flb_connection *connection, int (*callback)(void *data), int mask); diff --git a/src/flb_downstream.c b/src/flb_downstream.c index 23e44691f08..6b5f4ac6124 100644 --- a/src/flb_downstream.c +++ b/src/flb_downstream.c @@ -107,6 +107,24 @@ static void flb_downstream_conn_event_coro(void) flb_downstream_conn_event_loop(connection, coro); } +/* Complete the setup of a socket that was accepted outside this coroutine */ +static int downstream_conn_adopted_setup(struct flb_connection *connection, + struct flb_coro *coro) +{ +#ifdef FLB_HAVE_TLS + if (flb_stream_is_secure(connection->stream) && + connection->stream->tls_context != NULL) { + return flb_tls_session_create(connection->stream->tls_context, + connection, coro); + } +#else + (void) connection; + (void) coro; +#endif + + return 0; +} + static void flb_downstream_conn_accept_event_coro(void) { int result; @@ -117,7 +135,13 @@ static void flb_downstream_conn_accept_event_coro(void) connection = coro->data; flb_connection_reset_connection_timeout(connection); - result = flb_io_net_accept(connection, coro); + if (connection->fd == FLB_INVALID_SOCKET) { + result = flb_io_net_accept(connection, coro); + } + else { + /* the socket was accepted by another worker and handed over */ + result = downstream_conn_adopted_setup(connection, coro); + } flb_connection_unset_connection_timeout(connection); if (result == 0 && @@ -721,8 +745,14 @@ int flb_downstream_conn_release_all(struct flb_downstream *stream) return 0; } -int flb_downstream_conn_event_accept( +/* + * Start the accept coroutine of a connection. With an invalid 'fd' the + * coroutine accepts from the listener, otherwise it adopts 'fd'. An adopted + * socket is owned by the connection once this function is entered. + */ +static int downstream_conn_event_start( struct flb_downstream *stream, + flb_sockfd_t fd, flb_connection_accept_callback accept_callback, void *accept_callback_data, flb_connection_event_callback event_callback, @@ -739,10 +769,18 @@ int flb_downstream_conn_event_accept( (mask & (MK_EVENT_READ | MK_EVENT_WRITE)) == 0 || (stream->base.transport != FLB_TRANSPORT_TCP && stream->base.transport != FLB_TRANSPORT_UNIX_STREAM)) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } return -1; } if (stream->paused == FLB_TRUE) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + return 0; + } + connection_fd = flb_net_accept(stream->server_fd); if (connection_fd >= 0) { flb_socket_close(connection_fd); @@ -755,20 +793,29 @@ int flb_downstream_conn_event_accept( config = stream->base.config; if (config == NULL || flb_downstream_is_async(stream) == FLB_FALSE) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } return -1; } - connection = flb_connection_create(FLB_INVALID_SOCKET, + connection = flb_connection_create(fd, FLB_DOWNSTREAM_CONNECTION, stream, flb_engine_evl_get(), NULL); if (connection == NULL) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } return -1; } coro = flb_coro_create(connection); if (coro == NULL) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } flb_connection_destroy(connection); return -1; } @@ -779,6 +826,9 @@ int flb_downstream_conn_event_accept( &stack_size); if (coro->callee == NULL) { flb_coro_destroy(coro); + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } flb_connection_destroy(connection); return -1; } @@ -813,6 +863,35 @@ int flb_downstream_conn_event_accept( return ret; } +int flb_downstream_conn_event_accept( + struct flb_downstream *stream, + flb_connection_accept_callback accept_callback, + void *accept_callback_data, + flb_connection_event_callback event_callback, + int mask) +{ + return downstream_conn_event_start(stream, FLB_INVALID_SOCKET, + accept_callback, accept_callback_data, + event_callback, mask); +} + +int flb_downstream_conn_event_adopt( + struct flb_downstream *stream, + flb_sockfd_t fd, + flb_connection_accept_callback accept_callback, + void *accept_callback_data, + flb_connection_event_callback event_callback, + int mask) +{ + if (fd == FLB_INVALID_SOCKET) { + return -1; + } + + return downstream_conn_event_start(stream, fd, + accept_callback, accept_callback_data, + event_callback, mask); +} + int flb_downstream_conn_event_register(struct flb_connection *connection, flb_connection_event_callback callback, int mask) From a60c2f357ea490f8cfa575045946cc4a7100ceea Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 23:05:05 -0600 Subject: [PATCH 21/33] http_server: balance accepted connections across workers With SO_REUSEPORT the kernel hashes each connection onto one worker. A few long-lived keep-alive clients can therefore saturate one worker while the others stay idle. When several workers are configured, the worker that accepts a connection now hands it to the worker with the fewest active and pending connections through a per-worker queue and wake-up pipe. The max_connections slot is reserved before the handoff so admission limits stay exact. Signed-off-by: Eduardo Silva --- .../fluent-bit/http_server/flb_http_server.h | 7 + src/http_server/flb_http_server.c | 372 ++++++++++++++++++ 2 files changed, 379 insertions(+) diff --git a/include/fluent-bit/http_server/flb_http_server.h b/include/fluent-bit/http_server/flb_http_server.h index 07f4bc98f4f..d65e4729e19 100755 --- a/include/fluent-bit/http_server/flb_http_server.h +++ b/include/fluent-bit/http_server/flb_http_server.h @@ -103,6 +103,9 @@ struct flb_http_server_options { flb_http_server_worker_callback cb_worker_exit; }; +struct flb_http_server_balancer; +struct flb_http_server_handoff; + struct flb_http_server { /* Internal */ struct mk_event listener_event; @@ -138,6 +141,9 @@ struct flb_http_server { flb_http_server_worker_callback cb_worker_init; flb_http_server_worker_callback cb_worker_exit; struct flb_downstream_worker_runtime *runtime; + /* connection balancing between managed workers */ + struct flb_http_server_balancer *balancer; + struct flb_http_server_handoff *handoff; }; struct flb_http_server_session { @@ -156,6 +162,7 @@ struct flb_http_server_session { int drop_pending; int destroying; int connection_slot_reserved; + int balanced_connection; struct flb_connection *connection; struct flb_http_server *parent; diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index 19b83452ef4..eba86b7a5f1 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -41,6 +42,32 @@ struct flb_http_server_worker_context { struct flb_net_setup net_setup; }; +/* + * Connection balancing between managed workers. Every worker accepts from + * its own SO_REUSEPORT listener and the kernel spreads connections over + * those listeners by hash, so a few long-lived connections can pile up on one + * worker while the others stay idle. The accepting worker therefore hands + * each new socket to the worker that owns the fewest connections. + */ +struct flb_http_server_handoff { + struct mk_event event; /* must be the first member */ + pthread_mutex_t lock; + uint64_t ready; + flb_pipefd_t channel[2]; + flb_sockfd_t *queue; + size_t queue_length; + size_t queue_size; + uint64_t connections; /* sessions owned by the worker */ + uint64_t pending; /* sockets waiting in the queue */ + struct flb_http_server *server; +}; + +struct flb_http_server_balancer { + int workers; + uint64_t next; + struct flb_http_server_handoff *slots; +}; + static void flb_http_server_runtime_stop(struct flb_http_server *session); static void flb_http_server_pause_on_event_loop(struct flb_http_server *server); @@ -116,6 +143,23 @@ static void flb_http_server_reap_stale_sessions(struct flb_http_server *server) } } +static void flb_http_server_counter_add(uint64_t *counter, int64_t delta) +{ + uint64_t value; + + while (FLB_TRUE) { + value = cfl_atomic_load(counter); + + if (delta < 0 && value < (uint64_t) -delta) { + return; + } + + if (cfl_atomic_compare_exchange(counter, value, value + delta)) { + return; + } + } +} + static int flb_http_server_connection_slot_reserve(struct flb_http_server *server) { uint64_t connection_count; @@ -204,6 +248,8 @@ static int flb_http_server_apply_options(struct flb_http_server *session, session->cb_worker_init = options->cb_worker_init; session->cb_worker_exit = options->cb_worker_exit; session->runtime = NULL; + session->balancer = NULL; + session->handoff = NULL; cfl_list_init(&session->clients); cfl_list_init(&session->detached_clients); @@ -566,6 +612,11 @@ static int flb_http_server_client_connection_initialize( session->connection = connection; session->connection_slot_reserved = FLB_TRUE; + if (server->handoff != NULL) { + flb_http_server_counter_add(&server->handoff->connections, 1); + session->balanced_connection = FLB_TRUE; + } + if (session->version <= HTTP_PROTOCOL_VERSION_11) { session->http1.stream.user_data = server->user_data; } @@ -591,6 +642,172 @@ static int flb_http_server_client_connection_initialize( return 0; } +static int flb_http_server_connection_adopt(struct flb_http_server *server, + flb_sockfd_t fd) +{ + return flb_downstream_conn_event_adopt(server->downstream, fd, + flb_http_server_client_connection_initialize, + server, + flb_http_server_client_activity_event_handler, + MK_EVENT_READ); +} + +/* Worker owning the fewest connections, rotating the start to break ties */ +static struct flb_http_server_handoff *flb_http_server_balancer_pick( + struct flb_http_server_balancer *balancer) +{ + int i; + int index; + int start; + uint64_t load; + uint64_t best_load; + struct flb_http_server_handoff *slot; + struct flb_http_server_handoff *best; + + best = NULL; + best_load = 0; + start = (int) (cfl_atomic_load(&balancer->next) % balancer->workers); + flb_http_server_counter_add(&balancer->next, 1); + + for (i = 0; i < balancer->workers; i++) { + index = (start + i) % balancer->workers; + slot = &balancer->slots[index]; + + if (cfl_atomic_load(&slot->ready) == FLB_FALSE) { + continue; + } + + load = cfl_atomic_load(&slot->connections) + cfl_atomic_load(&slot->pending); + if (best == NULL || load < best_load) { + best = slot; + best_load = load; + } + } + + return best; +} + +/* Queue a socket for another worker, the socket is not consumed on failure */ +static int flb_http_server_handoff_push(struct flb_http_server_handoff *slot, + flb_sockfd_t fd) +{ + char signal; + size_t size; + flb_sockfd_t *queue; + + pthread_mutex_lock(&slot->lock); + + if (slot->ready == FLB_FALSE) { + pthread_mutex_unlock(&slot->lock); + return -1; + } + + if (slot->queue_length == slot->queue_size) { + size = slot->queue_size == 0 ? 16 : slot->queue_size * 2; + queue = flb_realloc(slot->queue, size * sizeof(flb_sockfd_t)); + if (queue == NULL) { + flb_errno(); + pthread_mutex_unlock(&slot->lock); + return -1; + } + slot->queue = queue; + slot->queue_size = size; + } + + /* one notification per empty to non-empty transition */ + if (slot->queue_length == 0) { + signal = 1; + if (flb_pipe_w(slot->channel[1], &signal, sizeof(signal)) != sizeof(signal)) { + pthread_mutex_unlock(&slot->lock); + return -1; + } + } + + slot->queue[slot->queue_length++] = fd; + flb_http_server_counter_add(&slot->pending, 1); + + pthread_mutex_unlock(&slot->lock); + + return 0; +} + +/* Runs on the worker owning 'slot' when sockets were queued for it */ +static int flb_http_server_handoff_event_handler(void *data) +{ + char signal; + size_t i; + size_t length; + flb_sockfd_t *queue; + struct flb_http_server_handoff *slot; + + slot = (struct flb_http_server_handoff *) data; + + if (flb_pipe_r(slot->channel[0], &signal, sizeof(signal)) != sizeof(signal)) { + return -1; + } + + pthread_mutex_lock(&slot->lock); + queue = slot->queue; + length = slot->queue_length; + slot->queue = NULL; + slot->queue_length = 0; + slot->queue_size = 0; + flb_http_server_counter_add(&slot->pending, -((int64_t) length)); + pthread_mutex_unlock(&slot->lock); + + for (i = 0; i < length; i++) { + /* the session reserves the slot again while it is set up */ + flb_http_server_connection_slot_release(slot->server); + flb_http_server_connection_adopt(slot->server, queue[i]); + } + + if (queue != NULL) { + flb_free(queue); + } + + return 0; +} + +/* Accept from this worker listener and give each socket to the least loaded worker */ +static int flb_http_server_balanced_accept(struct flb_http_server *server) +{ + int accepted; + flb_sockfd_t fd; + struct flb_http_server_handoff *target; + + for (accepted = 0; accepted < FLB_HTTP_SERVER_ACCEPT_BATCH_SIZE; accepted++) { + fd = flb_net_accept(server->downstream->server_fd); + if (fd == FLB_INVALID_SOCKET) { + break; + } + + if (server->downstream->paused == FLB_TRUE) { + flb_socket_close(fd); + continue; + } + + target = flb_http_server_balancer_pick(server->balancer); + + /* + * A connection limit slot is held while the socket waits for the + * other worker, so connections are still admitted in accept order. + * When no slot is free the socket stays here and the session setup + * rejects it as usual. + */ + if (target != NULL && target != server->handoff && + flb_http_server_connection_slot_reserve(server)) { + if (flb_http_server_handoff_push(target, fd) == 0) { + continue; + } + flb_http_server_connection_slot_release(server); + } + + flb_http_server_connection_adopt(server, fd); + } + + return 0; +} + static int flb_http_server_client_connection_event_handler(void *data) { int accepted_connections; @@ -626,6 +843,10 @@ static int flb_http_server_client_connection_event_handler(void *data) return result; } + if (server->balancer != NULL && server->handoff != NULL) { + return flb_http_server_balanced_accept(server); + } + accepted_connections = 0; do { @@ -664,6 +885,129 @@ static void flb_http_server_worker_maintenance(struct flb_downstream_worker *wor flb_http_server_reap_stale_sessions(&context->server); } +static void flb_http_server_balancer_destroy(struct flb_http_server_balancer *balancer) +{ + int i; + + if (balancer == NULL) { + return; + } + + for (i = 0; i < balancer->workers; i++) { + pthread_mutex_destroy(&balancer->slots[i].lock); + } + + flb_free(balancer->slots); + flb_free(balancer); +} + +static struct flb_http_server_balancer *flb_http_server_balancer_create(int workers) +{ + int i; + struct flb_http_server_balancer *balancer; + + balancer = flb_calloc(1, sizeof(struct flb_http_server_balancer)); + if (balancer == NULL) { + flb_errno(); + return NULL; + } + + balancer->slots = flb_calloc(workers, sizeof(struct flb_http_server_handoff)); + if (balancer->slots == NULL) { + flb_errno(); + flb_free(balancer); + return NULL; + } + + for (i = 0; i < workers; i++) { + if (pthread_mutex_init(&balancer->slots[i].lock, NULL) != 0) { + balancer->workers = i; + flb_http_server_balancer_destroy(balancer); + return NULL; + } + balancer->slots[i].channel[0] = -1; + balancer->slots[i].channel[1] = -1; + } + balancer->workers = workers; + + return balancer; +} + +/* Register the handoff channel of a worker, runs on the worker thread */ +static int flb_http_server_handoff_attach(struct flb_http_server *server, + struct flb_http_server_balancer *balancer, + int worker_id) +{ + int result; + struct flb_http_server_handoff *slot; + + if (worker_id < 0 || worker_id >= balancer->workers) { + return -1; + } + + slot = &balancer->slots[worker_id]; + + result = flb_pipe_create(slot->channel); + if (result != 0) { + return -1; + } + + MK_EVENT_NEW(&slot->event); + slot->event.type = FLB_ENGINE_EV_CUSTOM; + slot->event.handler = flb_http_server_handoff_event_handler; + + result = mk_event_add(server->event_loop, slot->channel[0], + FLB_ENGINE_EV_CUSTOM, MK_EVENT_READ, &slot->event); + if (result == -1) { + flb_pipe_destroy(slot->channel); + slot->channel[0] = -1; + slot->channel[1] = -1; + return -1; + } + + slot->server = server; + server->balancer = balancer; + server->handoff = slot; + + pthread_mutex_lock(&slot->lock); + cfl_atomic_store(&slot->ready, FLB_TRUE); + pthread_mutex_unlock(&slot->lock); + + return 0; +} + +/* Stop receiving sockets and close the ones still queued, runs on the worker thread */ +static void flb_http_server_handoff_detach(struct flb_http_server *server) +{ + size_t i; + struct flb_http_server_handoff *slot; + + slot = server->handoff; + if (slot == NULL) { + return; + } + + pthread_mutex_lock(&slot->lock); + cfl_atomic_store(&slot->ready, FLB_FALSE); + for (i = 0; i < slot->queue_length; i++) { + flb_http_server_connection_slot_release(server); + flb_socket_close(slot->queue[i]); + } + if (slot->queue != NULL) { + flb_free(slot->queue); + } + slot->queue = NULL; + slot->queue_length = 0; + slot->queue_size = 0; + cfl_atomic_store(&slot->pending, 0); + pthread_mutex_unlock(&slot->lock); + + mk_event_del(server->event_loop, &slot->event); + flb_pipe_destroy(slot->channel); + slot->channel[0] = -1; + slot->channel[1] = -1; +} + static int flb_http_server_worker_initialize(struct flb_downstream_worker *worker, struct flb_http_server *parent, struct flb_http_server_worker_context *context) @@ -718,6 +1062,15 @@ static int flb_http_server_worker_initialize(struct flb_downstream_worker *worke context->server.worker_id = flb_downstream_worker_id_get(worker); context->server.workers = parent->workers; + if (parent->balancer != NULL) { + result = flb_http_server_handoff_attach(&context->server, parent->balancer, + context->server.worker_id); + if (result != 0) { + flb_http_server_destroy(&context->server); + return result; + } + } + return 0; } @@ -746,6 +1099,7 @@ static int flb_http_server_worker_init(struct flb_downstream_worker *worker, ret = flb_downstream_worker_listener_fd_set( worker, context->server.downstream->server_fd); if (ret != 0) { + flb_http_server_handoff_detach(&context->server); flb_http_server_destroy(&context->server); flb_free(context); return ret; @@ -765,6 +1119,7 @@ static void flb_http_server_worker_exit(struct flb_downstream_worker *worker, context = worker_context; + flb_http_server_handoff_detach(&context->server); flb_http_server_destroy(&context->server); flb_free(context); } @@ -787,6 +1142,13 @@ static int flb_http_server_runtime_start(struct flb_http_server *session) session->tls_alpn_configured = FLB_TRUE; } + if (session->workers > 1) { + session->balancer = flb_http_server_balancer_create(session->workers); + if (session->balancer == NULL) { + return -1; + } + } + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); options.workers = session->workers; options.parent = session; @@ -796,6 +1158,8 @@ static int flb_http_server_runtime_start(struct flb_http_server *session) result = flb_downstream_worker_runtime_start(&session->runtime, &options); if (result != 0) { + flb_http_server_balancer_destroy(session->balancer); + session->balancer = NULL; return -1; } @@ -808,6 +1172,9 @@ static void flb_http_server_runtime_stop(struct flb_http_server *session) { flb_downstream_worker_runtime_stop(session->runtime); session->runtime = NULL; + + flb_http_server_balancer_destroy(session->balancer); + session->balancer = NULL; } /* HTTP SERVER */ @@ -1358,6 +1725,11 @@ void flb_http_server_session_destroy(struct flb_http_server_session *session) session->connection_slot_reserved = FLB_FALSE; } + if (session->balanced_connection && session->parent->handoff != NULL) { + flb_http_server_counter_add(&session->parent->handoff->connections, -1); + session->balanced_connection = FLB_FALSE; + } + if (session->incoming_data != NULL) { cfl_sds_destroy(session->incoming_data); } From 65c52ca4b7a2ff13411080924802dfd2eb932cd1 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 23:05:06 -0600 Subject: [PATCH 22/33] tests: integration: cover balanced HTTP worker connections Signed-off-by: Eduardo Silva --- .../tests/test_in_opentelemetry_001.py | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py index 42684fcfa75..01843d9eed4 100644 --- a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py +++ b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py @@ -20,6 +20,7 @@ import time import base64 import gzip +import http.client from concurrent.futures import ThreadPoolExecutor import grpc import requests @@ -3133,6 +3134,56 @@ def test_in_opentelemetry_logs_direct_cfl_body_key(tmp_path, encoding): assert direct_records == msgpack_records +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +def test_in_opentelemetry_http_workers_balance_keepalive_connections(tmp_path, encoding): + # Every accepted socket may be handed to another worker. Many concurrent + # keep-alive connections must all be served and every record delivered. + connections = 24 + requests_per_connection = 10 + service = direct_cfl_service(tmp_path, True, DIRECT_CFL_PROCESSOR_CHAINS["hash_once"], + workers=4, threaded=True) + service.start() + try: + before = len(data_storage["logs"]) + payload = build_direct_cfl_sentinel_payload("keep") + if encoding == "json": + body = json_format.MessageToJson(payload).encode("utf-8") + content_type = "application/json" + else: + body = payload.SerializeToString() + content_type = "application/x-protobuf" + + def client(): + connection = http.client.HTTPConnection("127.0.0.1", service.flb_listener_port, + timeout=10) + try: + for _ in range(requests_per_connection): + connection.request("POST", "/v1/logs", body, + {"Content-Type": content_type}) + response = connection.getresponse() + response.read() + assert response.status == 201 + finally: + connection.close() + + with ThreadPoolExecutor(max_workers=connections) as pool: + for future in [pool.submit(client) for _ in range(connections)]: + future.result() + + expected = connections * requests_per_connection + + def delivered(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if len(records) >= expected else None + + records = service.service.wait_for_condition(delivered, timeout=30, interval=0.2, + description="balanced connections") + assert len(records) == expected + finally: + service.stop() + + @pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") def test_in_opentelemetry_logs_direct_cfl_workers(tmp_path): # Hashing makes accidental processing in both the worker and collector From a21ceff8d4a2b1709916d6d2598fe4c93c14be5c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 28 Oct 2026 23:05:06 -0600 Subject: [PATCH 23/33] benchmarks: compare OTLP log ingestion with the Collector Add a harness that runs Fluent Bit releases and the OpenTelemetry Collector against the same OTLP/HTTP JSON and protobuf workloads, with and without a record modification, and reports steady-state delivered throughput, CPU per record, memory and latency. Signed-off-by: Eduardo Silva --- benchmarks/compare_fluent_bit_otelcol.py | 712 +++++++++++++++++++++++ 1 file changed, 712 insertions(+) create mode 100644 benchmarks/compare_fluent_bit_otelcol.py diff --git a/benchmarks/compare_fluent_bit_otelcol.py b/benchmarks/compare_fluent_bit_otelcol.py new file mode 100644 index 00000000000..7d9ec969830 --- /dev/null +++ b/benchmarks/compare_fluent_bit_otelcol.py @@ -0,0 +1,712 @@ +#!/usr/bin/env python3 +"""Compare Fluent Bit and Collector OTLP/HTTP log ingestion to null sinks. + +Measurement model: + +- The load generator runs as one process per client so it is not limited by + the Python GIL. Its own CPU use is reported so a generator ceiling is + visible in the results. +- Server and generator CPU sets must not share a physical core; the harness + checks the SMT sibling map and refuses to run otherwise. It also refuses to + start while other processes keep those CPUs busy. +- Fluent Bit refreshes its metrics endpoints once per second, the Collector + serves live counters. To measure both the same way, throughput is the + slope of the completed-records counter over a steady-state window + (by default 10% to 90% of the measured records), using the time at which + each new counter value was first observed. End-to-end completion time is + still reported, but it is quantized to one second for Fluent Bit. +""" + +import argparse +import csv +import hashlib +import http.client +import json +import math +import multiprocessing +import os +from pathlib import Path +import re +import socket +import statistics +import subprocess +import tempfile +import threading +import time +import urllib.request + + +ENCODINGS = ("json", "protobuf") +PIPELINES = ("ingest", "modified") +SAMPLE_INTERVAL = 0.05 +CLIENT_DEADLINE = 900 +CLIENT_MAX_CONSECUTIVE_FAILURES = 200 + + +def free_port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def parse_cpus(text): + cpus = set() + for item in text.split(","): + if "-" in item: + low, high = item.split("-") + cpus.update(range(int(low), int(high) + 1)) + else: + cpus.add(int(item)) + return cpus + + +def physical_core(cpu): + path = Path(f"/sys/devices/system/cpu/cpu{cpu}/topology/thread_siblings_list") + return min(parse_cpus(path.read_text().strip())) + + +def check_topology(server_cpus, generator_cpus): + server_cores = {physical_core(cpu) for cpu in server_cpus} + generator_cores = {physical_core(cpu) for cpu in generator_cpus} + shared = server_cores & generator_cores + if shared: + raise SystemExit(f"server and generator CPUs share physical cores {sorted(shared)}") + + +def cpu_busy_fraction(cpus, interval=1.0): + def read(): + values = {} + for line in Path("/proc/stat").read_text().splitlines(): + if line.startswith("cpu") and line[3:4].isdigit(): + fields = line.split() + numbers = [int(value) for value in fields[1:]] + values[int(fields[0][3:])] = (numbers[3] + numbers[4], sum(numbers)) + return values + + before = read() + time.sleep(interval) + after = read() + idle = sum(after[cpu][0] - before[cpu][0] for cpu in cpus) + total = sum(after[cpu][1] - before[cpu][1] for cpu in cpus) + return 1.0 - idle / total if total else 0.0 + + +def wait_quiet(cpus, limit=0.10, attempts=30): + for _ in range(attempts): + busy = cpu_busy_fraction(cpus) + if busy <= limit: + return busy + raise RuntimeError(f"benchmark CPUs stayed busy ({busy:.0%}); stop other workloads first") + + +def percentile(values, percentile_value): + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, math.ceil(percentile_value * len(ordered)) - 1)) + return ordered[index] + + +def process_cpu_seconds(pid): + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + ticks = os.sysconf("SC_CLK_TCK") + return (int(fields[11]) + int(fields[12])) / ticks + + +def thread_cpu_ticks(pid): + threads = {} + for task in Path(f"/proc/{pid}/task").iterdir(): + try: + fields = (task / "stat").read_text().rsplit(")", 1)[1].split() + name = (task / "comm").read_text().strip() + except (FileNotFoundError, ProcessLookupError): + continue + threads[int(task.name)] = (name, int(fields[11]) + int(fields[12])) + return threads + + +def process_rss_bytes(pid): + for line in Path(f"/proc/{pid}/status").read_text().splitlines(): + if line.startswith("VmRSS:"): + return int(line.split()[1]) * 1024 + raise RuntimeError("VmRSS was absent") + + +def scrape(url): + with urllib.request.urlopen(url, timeout=2) as response: + return response.read().decode() + + +def prometheus_value(text, metric): + pattern = re.compile(rf"^{re.escape(metric)}(?:\{{[^}}]*\}})?\s+([0-9.eE+-]+)$", re.MULTILINE) + values = [float(match.group(1)) for match in pattern.finditer(text)] + return sum(values) + + +def fluent_metrics(port): + data = json.loads(scrape(f"http://127.0.0.1:{port}/api/v1/metrics")) + output = data.get("output", {}).get("null.0", {}) + input_metrics = data.get("input", {}).get("opentelemetry.0", {}) + return { + "accepted": int(input_metrics.get("records", 0)), + "completed": int(output.get("proc_records", 0)), + "dropped": int(output.get("dropped_records", 0)), + "errors": int(output.get("errors", 0)), + "retries": int(output.get("retries", 0)), + "queue": 0, + } + + +def collector_metrics(port): + text = scrape(f"http://127.0.0.1:{port}/metrics") + return { + "accepted": int(prometheus_value(text, "otelcol_receiver_accepted_log_records")), + "completed": int(prometheus_value(text, "otelcol_null_discarded_log_records")), + "dropped": int(prometheus_value(text, "otelcol_exporter_enqueue_failed_log_records")), + "errors": int(prometheus_value(text, "otelcol_receiver_refused_log_records")), + "retries": int(prometheus_value(text, "otelcol_exporter_send_failed_log_records")), + "queue": int(prometheus_value(text, "otelcol_exporter_queue_size")), + } + + +def wait_ready(metrics_function, port, process): + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("server exited during startup") + try: + metrics_function(port) + return + except Exception: + time.sleep(0.05) + raise TimeoutError("metrics endpoint did not become ready") + + +def wait_completed(metrics_function, port, process, expected, timeout): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("server exited before drain") + try: + current = metrics_function(port) + except Exception: + # Fluent Bit answers 404 while no metrics snapshot is published + time.sleep(0.01) + continue + if current["completed"] >= expected and current["queue"] == 0: + if current["completed"] != expected: + raise RuntimeError(f"completion exceeded expected count: {current}") + return current + time.sleep(0.01) + raise TimeoutError(f"completion did not reach {expected}") + + +def make_payloads(batch_records): + attributes = [ + {"key": f"field{index}", "value": {"stringValue": "representative attribute"}} + for index in range(10) + ] + log_record = { + "timeUnixNano": "1700000000000000000", + "severityNumber": 9, + "severityText": "INFO", + "body": {"stringValue": "representative benchmark log message"}, + "attributes": attributes, + } + request = { + "resourceLogs": [{ + "resource": {"attributes": [ + {"key": "service.name", "value": {"stringValue": "benchmark"}} + ]}, + "scopeLogs": [{ + "scope": {"name": "benchmark", "version": "1.0"}, + "logRecords": [log_record] * batch_records, + }], + }] + } + json_payload = json.dumps(request, separators=(",", ":")).encode() + from google.protobuf.json_format import ParseDict + from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + protobuf_payload = ParseDict(request, ExportLogsServiceRequest()).SerializeToString() + return {"json": json_payload, "protobuf": protobuf_payload} + + +def system_kind(name): + if "otelcol" in name: + return "collector" + return "fluent-bit" + + +def fluent_config(input_port, metrics_port, modified, flush_seconds): + processor = "" + if modified: + processor = """ processors: + logs: + - name: content_modifier + context: otel_log_attributes + action: upsert + key: bench.added + value: yes +""" + return f"""service: + flush: {flush_seconds} + grace: 30 + log_level: error + http_server: on + http_listen: 127.0.0.1 + http_port: {metrics_port} +pipeline: + inputs: + - name: opentelemetry + tag: benchmark + listen: 127.0.0.1 + port: {input_port} + http2: off + tls: off + threaded: true + http_server.workers: 4 +{processor} outputs: + - name: null + match: '*' +""" + + +def collector_config(input_port, metrics_port, modified, flush_seconds): + processor_definition = "" + pipeline_processor = "" + if modified: + processor_definition = """processors: + transform/add_attribute: + error_mode: propagate + log_statements: + - context: log + statements: + - set(attributes[\"bench.added\"], \"yes\") + +""" + pipeline_processor = " processors: [transform/add_attribute]\n" + return f"""receivers: + otlp: + protocols: + http: + endpoint: 127.0.0.1:{input_port} + +{processor_definition}exporters: + null/fluent_bit: + sending_queue: + enabled: true + num_consumers: 1 + queue_size: 1000000 + sizer: items + batch: + enabled: true + flush_timeout: {flush_seconds}s + min_size: 8192 + sizer: items + +service: + telemetry: + logs: + level: error + metrics: + level: detailed + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: {metrics_port} + pipelines: + logs: + receivers: [otlp] +{pipeline_processor} exporters: [null/fluent_bit] +""" + + +def send_requests(task): + """Generator process body: one keep-alive connection, bounded retries.""" + port, payload, requests, content_type = task + latencies = [] + retries = 0 + failures = 0 + cpu_start = time.process_time() + deadline = time.monotonic() + CLIENT_DEADLINE + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30) + try: + for _ in range(requests): + started = time.monotonic_ns() + while True: + if time.monotonic() > deadline: + raise TimeoutError("client deadline exceeded") + try: + connection.request("POST", "/v1/logs", payload, + {"Content-Type": content_type}) + response = connection.getresponse() + body = response.read() + if response.status in (200, 201): + failures = 0 + break + if response.status not in (429, 503): + raise RuntimeError(f"HTTP {response.status}: {body!r}") + except (ConnectionError, TimeoutError, http.client.HTTPException, OSError): + connection.close() + failures += 1 + if failures >= CLIENT_MAX_CONSECUTIVE_FAILURES: + raise + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30) + retries += 1 + time.sleep(0.0005) + latencies.append((time.monotonic_ns() - started) / 1000) + finally: + connection.close() + return latencies, retries, time.process_time() - cpu_start + + +def run_clients(pool, clients, port, payload, requests_per_client, content_type): + tasks = [(port, payload, requests_per_client, content_type)] * clients + return pool.map(send_requests, tasks, chunksize=1) + + +def sample_process(pid, done, samples, metrics_function, metrics_port): + def take(): + samples.append((time.monotonic(), process_cpu_seconds(pid), + process_rss_bytes(pid), metrics_function(metrics_port), + thread_cpu_ticks(pid))) + + while not done.is_set(): + try: + take() + except FileNotFoundError: + return + except Exception: + pass + done.wait(SAMPLE_INTERVAL) + try: + take() + except Exception: + pass + + +def peak_cpu_cores(samples): + peak = 0.0 + for start_index, start in enumerate(samples): + for end in samples[start_index + 1:]: + elapsed = end[0] - start[0] + if elapsed >= 1.0: + peak = max(peak, (end[1] - start[1]) / elapsed) + break + return peak + + +def change_points(samples, key): + """Samples where the counter first shows a new value.""" + points = [] + previous = None + for sample in samples: + value = sample[3][key] + if value != previous: + points.append(sample) + previous = value + return points + + +def steady_window(samples, key, base, records, low, high): + points = change_points(samples, key) + lower = base + records * low + upper = base + records * high + start = next((point for point in points if point[3][key] >= lower), None) + end = next((point for point in reversed(points) if point[3][key] <= upper), None) + if start is None or end is None or end[0] - start[0] < 2.0: + raise RuntimeError(f"steady-state window for {key} is too short; raise --records") + return start, end + + +def thread_profile(start, end, top=4): + ticks = os.sysconf("SC_CLK_TCK") + elapsed = end[0] - start[0] + usage = {} + for tid, (name, value) in end[4].items(): + before = start[4].get(tid, (name, 0))[1] + usage[(tid, name)] = (value - before) / ticks / elapsed + ordered = sorted(usage.items(), key=lambda item: item[1], reverse=True) + busiest = ordered[0][1] if ordered else 0.0 + text = "|".join(f"{name}:{value:.2f}" for (tid, name), value in ordered[:top]) + return busiest, text + + +def run_once(system, binary, encoding, pipeline, repetition, args, payload, pool): + records = args.records + clients = args.clients + batch_records = args.batch_records + warmup_records = args.warmup_records + input_port = free_port() + metrics_port = free_port() + modified = pipeline == "modified" + kind = system_kind(system) + is_fluent_bit = kind != "collector" + metrics_function = fluent_metrics if is_fluent_bit else collector_metrics + if is_fluent_bit: + config_text = fluent_config(input_port, metrics_port, modified, args.flush_seconds) + else: + config_text = collector_config(input_port, metrics_port, modified, args.flush_seconds) + content_type = "application/json" if encoding == "json" else "application/x-protobuf" + quiet_busy = wait_quiet(args.server_cpu_set | args.generator_cpu_set) + with tempfile.TemporaryDirectory(prefix="otel-compare-") as directory: + config_path = Path(directory) / "config.yaml" + log_path = Path(directory) / "server.log" + config_path.write_text(config_text) + environment = os.environ.copy() + if not is_fluent_bit: + environment["GOMAXPROCS"] = str(len(args.server_cpu_set)) + with log_path.open("w+") as log: + process = subprocess.Popen( + ["taskset", "-c", args.server_cpus, str(binary), "--config", str(config_path)], + stdout=log, stderr=log, env=environment, + ) + try: + wait_ready(metrics_function, metrics_port, process) + run_clients(pool, clients, input_port, payload, + warmup_records // batch_records // clients, content_type) + wait_completed(metrics_function, metrics_port, process, warmup_records, 120) + idle_rss = process_rss_bytes(process.pid) + start_cpu = process_cpu_seconds(process.pid) + start = time.monotonic() + done = threading.Event() + samples = [] + monitor = threading.Thread(target=sample_process, + args=(process.pid, done, samples, + metrics_function, metrics_port), daemon=True) + monitor.start() + client_results = run_clients(pool, clients, input_port, payload, + records // batch_records // clients, + content_type) + send_end = time.monotonic() + final_metrics = wait_completed(metrics_function, metrics_port, process, + warmup_records + records, 600) + end = time.monotonic() + end_cpu = process_cpu_seconds(process.pid) + done.set() + monitor.join() + + accepted_delta = final_metrics["accepted"] - warmup_records + completed_delta = final_metrics["completed"] - warmup_records + if accepted_delta != records or completed_delta != records: + raise RuntimeError(f"accounting mismatch: {final_metrics}") + if final_metrics["dropped"] or final_metrics["errors"]: + raise RuntimeError(f"loss/refusal counters are nonzero: {final_metrics}") + + latencies = [latency for result in client_results for latency in result[0]] + retries = sum(result[1] for result in client_results) + generator_cpu = sum(result[2] for result in client_results) + samples = [sample for sample in samples if sample[3] is not None] + rss_values = [sample[2] for sample in samples] + peak_rss_sample = max(samples, key=lambda sample: sample[2]) + peak_backlog = max(sample[3]["accepted"] - sample[3]["completed"] + for sample in samples) + + completed_start, completed_end = steady_window( + samples, "completed", warmup_records, records, + args.window_low, args.window_high) + steady_seconds = completed_end[0] - completed_start[0] + steady_records = completed_end[3]["completed"] - completed_start[3]["completed"] + steady_cpu = completed_end[1] - completed_start[1] + busiest_thread, threads_text = thread_profile(completed_start, completed_end) + + accepted_start, accepted_end = steady_window( + samples, "accepted", warmup_records, records, + args.window_low, args.window_high) + steady_accepted_rate = ( + (accepted_end[3]["accepted"] - accepted_start[3]["accepted"]) / + (accepted_end[0] - accepted_start[0])) + + elapsed = end - start + send_elapsed = send_end - start + cpu = end_cpu - start_cpu + return { + "system": system, + "repetition": repetition, + "pipeline": pipeline, + "encoding": encoding, + "workers_or_gomaxprocs": len(args.server_cpu_set), + "clients": clients, + "batch_records": batch_records, + "payload_bytes": len(payload), + "payload_sha256": hashlib.sha256(payload).hexdigest(), + "offered_records": records, + "accepted_records": accepted_delta, + "completed_records": completed_delta, + "client_retries": retries, + "dropped_records": final_metrics["dropped"], + "refused_records": final_metrics["errors"], + "pre_run_cpu_busy": quiet_busy, + "send_seconds": send_elapsed, + "completion_seconds": elapsed, + "steady_window_seconds": steady_seconds, + "steady_completed_records_per_second": steady_records / steady_seconds, + "steady_accepted_records_per_second": steady_accepted_rate, + "steady_cpu_cores": steady_cpu / steady_seconds, + "steady_records_per_cpu_second": steady_records / steady_cpu, + "busiest_thread_cpu": busiest_thread, + "top_threads": threads_text, + "end_to_end_records_per_second": records / elapsed, + "payload_mib_per_second": (steady_records / batch_records * len(payload) / + steady_seconds / 1048576), + "cpu_seconds": cpu, + "cpu_seconds_per_million_records": cpu / records * 1_000_000, + "average_cpu_cores": cpu / elapsed, + "peak_cpu_cores_1s": peak_cpu_cores(samples), + "records_per_cpu_second": records / cpu, + "generator_cpu_seconds": generator_cpu, + "generator_cpu_utilization": (generator_cpu / send_elapsed / + len(args.generator_cpu_set)), + "idle_rss_mib": idle_rss / 1048576, + "steady_rss_mib": statistics.median(rss_values) / 1048576, + "peak_rss_mib": max(rss_values) / 1048576, + "peak_processing_backlog_records": peak_backlog, + "backlog_at_peak_rss_records": ( + peak_rss_sample[3]["accepted"] - + peak_rss_sample[3]["completed"]), + "latency_p50_us": percentile(latencies, 0.50), + "latency_p95_us": percentile(latencies, 0.95), + "latency_p99_us": percentile(latencies, 0.99), + "final_queue_items": final_metrics["queue"], + } + except Exception: + log.flush() + log.seek(0) + print(log.read(), flush=True) + raise + finally: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +SUMMARY_FIELDS = ( + "steady_completed_records_per_second", "steady_accepted_records_per_second", + "steady_cpu_cores", "steady_records_per_cpu_second", "busiest_thread_cpu", + "end_to_end_records_per_second", "payload_mib_per_second", + "cpu_seconds_per_million_records", "records_per_cpu_second", "average_cpu_cores", + "peak_cpu_cores_1s", "generator_cpu_utilization", + "idle_rss_mib", "steady_rss_mib", "peak_rss_mib", "peak_processing_backlog_records", + "latency_p50_us", "latency_p95_us", "latency_p99_us", +) + + +def summarize(rows, systems): + summary = [] + for pipeline in PIPELINES: + for encoding in ENCODINGS: + for system in systems: + selected = [row for row in rows if row["pipeline"] == pipeline + and row["encoding"] == encoding and row["system"] == system] + if not selected: + continue + output = { + "pipeline": pipeline, + "encoding": encoding, + "system": system, + "repetitions": len(selected), + } + for field in SUMMARY_FIELDS: + values = [float(row[field]) for row in selected] + output[f"median_{field}"] = statistics.median(values) + output[f"min_{field}"] = min(values) + output[f"max_{field}"] = max(values) + output["total_retries"] = sum(int(row["client_retries"]) for row in selected) + output["total_dropped"] = sum(int(row["dropped_records"]) for row in selected) + output["total_refused"] = sum(int(row["refused_records"]) for row in selected) + summary.append(output) + return summary + + +def write_csv(path, rows): + with path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +def output_path(prefix, suffix): + return Path(f"{prefix}{suffix}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--system", action="append", required=True, metavar="NAME=PATH", + help="binary to measure; names containing 'otelcol' run as the " + "Collector, any other name as Fluent Bit") + parser.add_argument("--output-prefix", type=Path, required=True) + parser.add_argument("--records", type=int, default=30_000_000) + parser.add_argument("--warmup-records", type=int, default=320_000) + parser.add_argument("--batch-records", type=int, default=100) + parser.add_argument("--clients", type=int, default=16) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--flush-seconds", type=float, default=0.5) + parser.add_argument("--server-cpus", default="0-3") + parser.add_argument("--generator-cpus", default="4-7,12-15") + parser.add_argument("--window-low", type=float, default=0.10) + parser.add_argument("--window-high", type=float, default=0.90) + parser.add_argument("--pipelines", default=",".join(PIPELINES)) + parser.add_argument("--encodings", default=",".join(ENCODINGS)) + args = parser.parse_args() + divisor = args.batch_records * args.clients + if args.records % divisor or args.warmup_records % divisor: + parser.error("record and warm-up counts must divide evenly across batches and clients") + + args.server_cpu_set = parse_cpus(args.server_cpus) + args.generator_cpu_set = parse_cpus(args.generator_cpus) + check_topology(args.server_cpu_set, args.generator_cpu_set) + os.sched_setaffinity(0, args.generator_cpu_set) + + binaries = {} + for item in args.system: + name, path = item.split("=", 1) + binaries[name] = Path(path).resolve() + systems = tuple(binaries) + pipelines = tuple(args.pipelines.split(",")) + encodings = tuple(args.encodings.split(",")) + + payloads = make_payloads(args.batch_records) + manifest = { + "created_unix": time.time(), + "server_cpus": sorted(args.server_cpu_set), + "generator_cpus": sorted(args.generator_cpu_set), + "arguments": {key: str(value) for key, value in vars(args).items()}, + "binaries": {system: str(path) for system, path in binaries.items()}, + "binary_sha256": { + system: hashlib.sha256(path.read_bytes()).hexdigest() + for system, path in binaries.items() + }, + "payloads": { + encoding: {"bytes": len(payload), + "sha256": hashlib.sha256(payload).hexdigest()} + for encoding, payload in payloads.items() + }, + "transformation": "upsert log attribute bench.added as string yes", + } + args.output_prefix.parent.mkdir(parents=True, exist_ok=True) + output_path(args.output_prefix, ".manifest.json").write_text( + json.dumps(manifest, indent=2, default=str) + "\n") + + rows = [] + context = multiprocessing.get_context("fork") + with context.Pool(args.clients) as pool: + for repetition in range(1, args.runs + 1): + for pipeline in pipelines: + for encoding in encodings: + offset = (repetition - 1) % len(systems) + order = systems[offset:] + systems[:offset] + for system in order: + row = run_once(system, binaries[system], encoding, pipeline, + repetition, args, payloads[encoding], pool) + rows.append(row) + write_csv(output_path(args.output_prefix, ".raw.csv"), rows) + print(json.dumps(row, sort_keys=True), flush=True) + write_csv(output_path(args.output_prefix, ".summary.csv"), summarize(rows, systems)) + + +if __name__ == "__main__": + main() From b457d8d9ccc16e244108ca97c1a7c48b6c204a23 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 29 Oct 2026 14:01:42 -0600 Subject: [PATCH 24/33] tests: internal: fuzzers: add OTLP logs decoder fuzzer Feed OTLP/HTTP log payloads to the JSON and protobuf decoders used by the OpenTelemetry input. The CFL decoders run followed by record iteration and encoding, then the msgpack decoders run on the same payload. The first input byte selects the encoding, the body key and the logs_max_size budget. Signed-off-by: Eduardo Silva --- tests/internal/fuzzers/CMakeLists.txt | 1 + .../fuzzers/opentelemetry_logs_fuzzer.c | 150 ++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 tests/internal/fuzzers/opentelemetry_logs_fuzzer.c diff --git a/tests/internal/fuzzers/CMakeLists.txt b/tests/internal/fuzzers/CMakeLists.txt index cb9bcdf0a77..b77812384b7 100644 --- a/tests/internal/fuzzers/CMakeLists.txt +++ b/tests/internal/fuzzers/CMakeLists.txt @@ -29,6 +29,7 @@ set(UNIT_TESTS_FILES config_map_fuzzer.c record_ac_fuzzer.c config_yaml_fuzzer.c + opentelemetry_logs_fuzzer.c ) # Prepare list of unit tests diff --git a/tests/internal/fuzzers/opentelemetry_logs_fuzzer.c b/tests/internal/fuzzers/opentelemetry_logs_fuzzer.c new file mode 100644 index 00000000000..70fd76f6ee2 --- /dev/null +++ b/tests/internal/fuzzers/opentelemetry_logs_fuzzer.c @@ -0,0 +1,150 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "flb_fuzz_header.h" + +/* + * Feed OTLP/HTTP log payloads to the decoders used by in_opentelemetry, the + * way the input does: the CFL decoders followed by the record encoding, and + * the msgpack decoders. The first byte selects the encoding, the body key and + * the logs_max_size budget, the rest is the request body. + */ + +/* Encode CFL records like the input does once processors are done */ +static void encode_chunk(struct flb_mp_chunk_cobj *chunk) +{ + struct flb_log_event_encoder encoder; + struct flb_mp_chunk_record *record; + + if (chunk == NULL) { + return; + } + + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + } + + if (flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) == 0) { + chunk->log_encoder = &encoder; + flb_mp_chunk_cobj_encode_append(chunk); + flb_mp_chunk_cobj_count_log_records(chunk); + chunk->log_encoder = NULL; + flb_log_event_encoder_destroy(&encoder); + } + + flb_mp_chunk_cobj_destroy(chunk); +} + +static void fuzz_protobuf(const char *payload, size_t size, + struct flb_opentelemetry_logs_decode_options *options) +{ + size_t record_count; + struct flb_log_event_encoder encoder; + struct flb_opentelemetry_logs_protobuf_cfl *result; + + result = NULL; + if (flb_opentelemetry_logs_protobuf_to_cfl(payload, size, options, &result) == 0) { + flb_opentelemetry_logs_protobuf_cfl_get_encoded_size(result); + encode_chunk(flb_opentelemetry_logs_protobuf_cfl_take_chunk(result)); + } + flb_opentelemetry_logs_protobuf_cfl_destroy(result); + + if (flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) == 0) { + encoder.output_limit = options->output_limit; + flb_opentelemetry_logs_protobuf_to_msgpack(&encoder, payload, size, options, + &record_count); + flb_log_event_encoder_destroy(&encoder); + } +} + +static void fuzz_json(const char *payload, size_t size, + struct flb_opentelemetry_logs_decode_options *options) +{ + int error_status; + struct flb_log_event_encoder encoder; + struct flb_opentelemetry_logs_json_cfl *result; + + result = NULL; + error_status = 0; + if (flb_opentelemetry_logs_json_to_cfl(payload, size, options, &result, + &error_status) == 0) { + flb_opentelemetry_logs_json_cfl_get_encoded_size(result); + encode_chunk(flb_opentelemetry_logs_json_cfl_take_chunk(result)); + } + flb_opentelemetry_logs_json_cfl_destroy(result); + + if (flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) == 0) { + encoder.output_limit = options->output_limit; + error_status = 0; + flb_opentelemetry_logs_json_to_msgpack(&encoder, payload, size, + options->body_key, &error_status); + flb_log_event_encoder_destroy(&encoder); + } +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint8_t selector; + char *payload; + struct flb_opentelemetry_logs_decode_options options; + + TIMEOUT_GUARD + + /* Set fuzzer-malloc chance of failure */ + flb_malloc_p = 0; + flb_malloc_mod = 25000; + + if (size < 1) { + return 0; + } + selector = data[0]; + MOVE_INPUT(1); + + /* exact-size copy, so reads past the payload are caught */ + payload = malloc(size > 0 ? size : 1); + if (payload == NULL) { + return 0; + } + if (size > 0) { + memcpy(payload, data, size); + } + + options.metadata_key = "otlp"; + options.body_key = (selector & 0x02) ? "log" : NULL; + options.output_limit = (selector >> 2) == 0 ? SIZE_MAX : (size_t) (selector >> 2) * 256; + + if (selector & 0x01) { + fuzz_protobuf(payload, size, &options); + } + else { + fuzz_json(payload, size, &options); + } + + free(payload); + + return 0; +} From 9ed77ea76209eb47a29a15eefae4669a27490d9c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 30 Oct 2026 01:32:09 -0600 Subject: [PATCH 25/33] in_opentelemetry: answer busy when processed logs hit a full queue When the input processors run in the HTTP workers, a full ingress queue was reported as a decoding failure and the client got a 400, so it did not resend the request. Return the busy status, which is answered with 503 like the unprocessed path. Signed-off-by: Eduardo Silva --- plugins/in_opentelemetry/opentelemetry_logs.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/in_opentelemetry/opentelemetry_logs.c b/plugins/in_opentelemetry/opentelemetry_logs.c index 2508d43504e..bbee1a38576 100644 --- a/plugins/in_opentelemetry/opentelemetry_logs.c +++ b/plugins/in_opentelemetry/opentelemetry_logs.c @@ -187,7 +187,11 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, } flb_opentelemetry_logs_protobuf_cfl_destroy(cfl_result); flb_opentelemetry_logs_json_cfl_destroy(json_cfl_result); - if (ret != 0) { + if (ret == FLB_INPUT_INGRESS_BUSY) { + /* the client retries once the ingress queue has room */ + return FLB_INPUT_INGRESS_BUSY; + } + else if (ret != 0) { flb_plg_error(ctx->ins, "failed to append logs to the input buffer"); return -1; } From 28c0110cc059de5d7d786bd304c947b07b8c4fc4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 30 Oct 2026 01:32:09 -0600 Subject: [PATCH 26/33] tests: integration: cover busy answers with worker processors Run the ingress queue byte limit case with a logs processor too, and send the oversized request as JSON and protobuf, both must get a 503. Signed-off-by: Eduardo Silva --- ...y_ingress_queue_byte_limit_processors.yaml | 31 +++++++++++++++++++ .../tests/test_in_opentelemetry_001.py | 21 +++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit_processors.yaml diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit_processors.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit_processors.yaml new file mode 100644 index 00000000000..8faa9775f00 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit_processors.yaml @@ -0,0 +1,31 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + http_server.workers: 4 + http_server.ingress_queue_event_limit: 1024 + http_server.ingress_queue_byte_limit: 2000 + processors: + logs: + - name: content_modifier + context: attributes + action: upsert + key: "processed" + value: "yes" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py index 01843d9eed4..1a66dd5b89e 100644 --- a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py +++ b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py @@ -74,6 +74,8 @@ IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_CONFIG = "otlp_http1_cleartext_workers_tiny_ingress_queue.yaml" IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_CONFIG = \ "otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit.yaml" +IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_PROCESSORS_CONFIG = \ + "otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit_processors.yaml" IN_OPENTELEMETRY_OAUTH2_PROTOCOL_CONFIGS = { "http1_cleartext": "otlp_http1_cleartext_oauth2.yaml", @@ -2147,8 +2149,16 @@ def test_in_opentelemetry_http_workers_respect_tiny_ingress_queue_limits(): ) == 0 -def test_in_opentelemetry_http_workers_respect_ingress_queue_byte_limit(): - service = Service(IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_CONFIG) +@pytest.mark.parametrize( + "config_file", + [ + IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_CONFIG, + IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_PROCESSORS_CONFIG, + ], + ids=["no_processors", "worker_processors"], +) +def test_in_opentelemetry_http_workers_respect_ingress_queue_byte_limit(config_file): + service = Service(config_file) small_payload = service.build_otel_payload("test_logs_001.in.json", "logs") small_request = ExportLogsServiceRequest() @@ -2159,6 +2169,7 @@ def test_in_opentelemetry_http_workers_respect_ingress_queue_byte_limit(): large_request.resource_logs.extend(small_request.resource_logs) large_payload = large_request.SerializeToString() + large_json_payload = json_format.MessageToJson(large_request).encode() assert len(small_payload) < len(large_payload) @@ -2176,6 +2187,12 @@ def test_in_opentelemetry_http_workers_respect_ingress_queue_byte_limit(): assert failure_response.status_code == 503 assert "deferred ingress queue is full" in failure_response.text + failure_response = service.send_raw_request( + "/v1/logs", large_json_payload, content_type="application/json" + ) + assert failure_response.status_code == 503 + assert "deferred ingress queue is full" in failure_response.text + metrics_text = service.service.wait_for_condition( lambda: ( metrics if maybe_read_prometheus_metric_value( From b028307c75e44bec5b834b796d319c51a8fb1ecb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 30 Oct 2026 01:32:09 -0600 Subject: [PATCH 27/33] benchmarks: add filesystem storage to the OTLP Collector comparison The --storage option buffers records in filesystem chunks on Fluent Bit and in a file_storage sending queue on the Collector. Requests the Collector refuses because its queue is full are resent by the clients, like Fluent Bit 503 answers, the delivered record accounting proves there is no loss. Signed-off-by: Eduardo Silva --- benchmarks/compare_fluent_bit_otelcol.py | 59 ++++++++++++++++++------ 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/benchmarks/compare_fluent_bit_otelcol.py b/benchmarks/compare_fluent_bit_otelcol.py index 7d9ec969830..c79e5df1e5e 100644 --- a/benchmarks/compare_fluent_bit_otelcol.py +++ b/benchmarks/compare_fluent_bit_otelcol.py @@ -235,8 +235,16 @@ def system_kind(name): return "fluent-bit" -def fluent_config(input_port, metrics_port, modified, flush_seconds): +def fluent_config(input_port, metrics_port, modified, flush_seconds, storage_dir): processor = "" + service_storage = "" + input_storage = "" + if storage_dir: + service_storage = f""" storage.path: {storage_dir} + storage.sync: normal + storage.checksum: off +""" + input_storage = " storage.type: filesystem\n" if modified: processor = """ processors: logs: @@ -253,7 +261,7 @@ def fluent_config(input_port, metrics_port, modified, flush_seconds): http_server: on http_listen: 127.0.0.1 http_port: {metrics_port} -pipeline: +{service_storage}pipeline: inputs: - name: opentelemetry tag: benchmark @@ -263,14 +271,25 @@ def fluent_config(input_port, metrics_port, modified, flush_seconds): tls: off threaded: true http_server.workers: 4 -{processor} outputs: +{input_storage}{processor} outputs: - name: null match: '*' """ -def collector_config(input_port, metrics_port, modified, flush_seconds): +def collector_config(input_port, metrics_port, modified, flush_seconds, storage_dir): processor_definition = "" + extension_definition = "" + queue_storage = "" + service_extensions = "" + if storage_dir: + extension_definition = f"""extensions: + file_storage: + directory: {storage_dir} + +""" + queue_storage = " storage: file_storage\n" + service_extensions = " extensions: [file_storage]\n" pipeline_processor = "" if modified: processor_definition = """processors: @@ -289,21 +308,21 @@ def collector_config(input_port, metrics_port, modified, flush_seconds): http: endpoint: 127.0.0.1:{input_port} -{processor_definition}exporters: +{extension_definition}{processor_definition}exporters: null/fluent_bit: sending_queue: enabled: true num_consumers: 1 queue_size: 1000000 sizer: items - batch: +{queue_storage} batch: enabled: true flush_timeout: {flush_seconds}s min_size: 8192 sizer: items service: - telemetry: +{service_extensions} telemetry: logs: level: error metrics: @@ -443,13 +462,19 @@ def run_once(system, binary, encoding, pipeline, repetition, args, payload, pool kind = system_kind(system) is_fluent_bit = kind != "collector" metrics_function = fluent_metrics if is_fluent_bit else collector_metrics - if is_fluent_bit: - config_text = fluent_config(input_port, metrics_port, modified, args.flush_seconds) - else: - config_text = collector_config(input_port, metrics_port, modified, args.flush_seconds) content_type = "application/json" if encoding == "json" else "application/x-protobuf" quiet_busy = wait_quiet(args.server_cpu_set | args.generator_cpu_set) - with tempfile.TemporaryDirectory(prefix="otel-compare-") as directory: + with tempfile.TemporaryDirectory(prefix="otel-compare-", dir=args.work_dir) as directory: + storage_dir = "" + if args.storage == "filesystem": + storage_dir = str(Path(directory) / "storage") + os.mkdir(storage_dir) + if is_fluent_bit: + config_text = fluent_config(input_port, metrics_port, modified, + args.flush_seconds, storage_dir) + else: + config_text = collector_config(input_port, metrics_port, modified, + args.flush_seconds, storage_dir) config_path = Path(directory) / "config.yaml" log_path = Path(directory) / "server.log" config_path.write_text(config_text) @@ -490,7 +515,9 @@ def run_once(system, binary, encoding, pipeline, repetition, args, payload, pool completed_delta = final_metrics["completed"] - warmup_records if accepted_delta != records or completed_delta != records: raise RuntimeError(f"accounting mismatch: {final_metrics}") - if final_metrics["dropped"] or final_metrics["errors"]: + # A full Collector queue refuses requests, the clients resend them + # like Fluent Bit 429/503 answers; the accounting above proves no loss + if is_fluent_bit and (final_metrics["dropped"] or final_metrics["errors"]): raise RuntimeError(f"loss/refusal counters are nonzero: {final_metrics}") latencies = [latency for result in client_results for latency in result[0]] @@ -651,6 +678,12 @@ def main(): parser.add_argument("--window-high", type=float, default=0.90) parser.add_argument("--pipelines", default=",".join(PIPELINES)) parser.add_argument("--encodings", default=",".join(ENCODINGS)) + parser.add_argument("--storage", choices=("memory", "filesystem"), default="memory", + help="buffer records in memory, or in filesystem chunks (Fluent Bit) " + "and a persistent sending queue (Collector file_storage)") + parser.add_argument("--work-dir", type=Path, default=None, + help="directory for configurations and filesystem storage " + "(default: the system temporary directory)") args = parser.parse_args() divisor = args.batch_records * args.clients if args.records % divisor or args.warmup_records % divisor: From 61eea2706fbde6e9d5bc0155947281251cf5ce84 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 30 Oct 2026 23:30:42 -0600 Subject: [PATCH 28/33] tests: internal: read the protobuf parity corpus length portably The request length prefix is little-endian, decode it byte by byte so the corpus is also read on big-endian hosts such as s390x. Signed-off-by: Eduardo Silva --- tests/internal/opentelemetry.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 5625a6e93a4..fa40f2262ce 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -2060,6 +2060,7 @@ void test_opentelemetry_protobuf_cfl_parity_corpus() size_t limit; size_t record_count; uint32_t length; + unsigned char prefix[4]; FILE *file; struct flb_log_event_encoder enc; struct flb_opentelemetry_logs_decode_options options; @@ -2076,7 +2077,9 @@ void test_opentelemetry_protobuf_cfl_parity_corpus() count = 0; /* each request is prefixed by its little-endian 32-bit length */ - while (fread(&length, sizeof(length), 1, file) == 1) { + while (fread(prefix, sizeof(prefix), 1, file) == 1) { + length = (uint32_t) prefix[0] | ((uint32_t) prefix[1] << 8) | + ((uint32_t) prefix[2] << 16) | ((uint32_t) prefix[3] << 24); payload = flb_malloc(length + 1); TEST_CHECK(payload != NULL); if (payload == NULL || fread(payload, 1, length, file) != length) { From 58b33fa81c6d46b7df9c420452e6b447a8422b09 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 31 Oct 2026 11:11:45 -0600 Subject: [PATCH 29/33] json: add an inline read-only tree for hot path traversal flb_json_tree reads a JSON document once and exposes its values through flb_json_node accessors, so callers traverse it without using the JSON backend directly. The accessors are inline and cost the same as the backend, the tree borrows its strings and replaces invalid Unicode like flb_pack_json() does. Without yyjson no tree is read and callers keep their other way to read JSON. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_json_tree.h | 441 +++++++++++++++++++++++++++++ src/flb_json.c | 37 +++ 2 files changed, 478 insertions(+) create mode 100644 include/fluent-bit/flb_json_tree.h diff --git a/include/fluent-bit/flb_json_tree.h b/include/fluent-bit/flb_json_tree.h new file mode 100644 index 00000000000..7db569621f9 --- /dev/null +++ b/include/fluent-bit/flb_json_tree.h @@ -0,0 +1,441 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_JSON_TREE_H +#define FLB_JSON_TREE_H + +#include +#include + +#include +#include +#include + +#ifdef FLB_HAVE_YYJSON +#include +#endif + +/* + * Read-only JSON tree for traversal in hot paths. A tree is parsed once and its + * strings are borrowed from it, no value is copied or converted. Invalid + * Unicode is replaced like flb_pack_json() does. flb_json_tree_read() returns + * NULL when the input is not a single JSON value or when no tree backend is + * built (FLB_YYJSON=Off), callers must keep another way to read the input. + * + * The accessors are inline so traversal costs the same as using the backend + * directly. They accept NULL and nodes of any type: type checks are false, + * sizes and numbers are zero and strings are NULL. + */ +struct flb_json_tree; +struct flb_json_node; + +enum flb_json_node_type { + FLB_JSON_NODE_NONE = 0, + FLB_JSON_NODE_NULL, + FLB_JSON_NODE_BOOL, + FLB_JSON_NODE_UINT, + FLB_JSON_NODE_SINT, + FLB_JSON_NODE_REAL, + FLB_JSON_NODE_STR, + FLB_JSON_NODE_ARR, + FLB_JSON_NODE_OBJ +}; + +struct flb_json_tree *flb_json_tree_read(const char *input, size_t length); +void flb_json_tree_destroy(struct flb_json_tree *tree); + +#ifdef FLB_HAVE_YYJSON + +static inline yyjson_val *flb_json_node_val(struct flb_json_node *node) +{ + return (yyjson_val *) node; +} + +static inline struct flb_json_node *flb_json_node_wrap(yyjson_val *value) +{ + return (struct flb_json_node *) value; +} + +static inline struct flb_json_node *flb_json_tree_root(struct flb_json_tree *tree) +{ + return flb_json_node_wrap(yyjson_doc_get_root((yyjson_doc *) tree)); +} + +/* one of enum flb_json_node_type, FLB_JSON_NODE_NONE for NULL */ +static inline int flb_json_node_type(struct flb_json_node *node) +{ + yyjson_val *value; + + value = flb_json_node_val(node); + switch (yyjson_get_type(value)) { + case YYJSON_TYPE_NULL: + return FLB_JSON_NODE_NULL; + case YYJSON_TYPE_BOOL: + return FLB_JSON_NODE_BOOL; + case YYJSON_TYPE_NUM: + switch (yyjson_get_subtype(value)) { + case YYJSON_SUBTYPE_UINT: + return FLB_JSON_NODE_UINT; + case YYJSON_SUBTYPE_SINT: + return FLB_JSON_NODE_SINT; + default: + return FLB_JSON_NODE_REAL; + } + case YYJSON_TYPE_STR: + return FLB_JSON_NODE_STR; + case YYJSON_TYPE_ARR: + return FLB_JSON_NODE_ARR; + case YYJSON_TYPE_OBJ: + return FLB_JSON_NODE_OBJ; + default: + return FLB_JSON_NODE_NONE; + } +} + +static inline int flb_json_node_is_null(struct flb_json_node *node) +{ + return yyjson_is_null(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_bool(struct flb_json_node *node) +{ + return yyjson_is_bool(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_uint(struct flb_json_node *node) +{ + return yyjson_is_uint(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_sint(struct flb_json_node *node) +{ + return yyjson_is_sint(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_int(struct flb_json_node *node) +{ + return yyjson_is_int(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_real(struct flb_json_node *node) +{ + return yyjson_is_real(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_str(struct flb_json_node *node) +{ + return yyjson_is_str(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_arr(struct flb_json_node *node) +{ + return yyjson_is_arr(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_obj(struct flb_json_node *node) +{ + return yyjson_is_obj(flb_json_node_val(node)); +} + +static inline int flb_json_node_get_bool(struct flb_json_node *node) +{ + return yyjson_get_bool(flb_json_node_val(node)); +} + +static inline uint64_t flb_json_node_get_uint(struct flb_json_node *node) +{ + return yyjson_get_uint(flb_json_node_val(node)); +} + +static inline int64_t flb_json_node_get_sint(struct flb_json_node *node) +{ + return yyjson_get_sint(flb_json_node_val(node)); +} + +static inline double flb_json_node_get_real(struct flb_json_node *node) +{ + return yyjson_get_real(flb_json_node_val(node)); +} + +static inline const char *flb_json_node_get_str(struct flb_json_node *node) +{ + return yyjson_get_str(flb_json_node_val(node)); +} + +static inline size_t flb_json_node_get_len(struct flb_json_node *node) +{ + return yyjson_get_len(flb_json_node_val(node)); +} + +/* string and its length, or NULL and a zero length when node is not a string */ +static inline const char *flb_json_node_get_strn(struct flb_json_node *node, + size_t *length) +{ + yyjson_val *value; + + value = flb_json_node_val(node); + if (!yyjson_is_str(value)) { + *length = 0; + return NULL; + } + + *length = unsafe_yyjson_get_len(value); + return unsafe_yyjson_get_str(value); +} + +static inline size_t flb_json_node_arr_size(struct flb_json_node *node) +{ + return yyjson_arr_size(flb_json_node_val(node)); +} + +static inline size_t flb_json_node_obj_size(struct flb_json_node *node) +{ + return yyjson_obj_size(flb_json_node_val(node)); +} + +/* first element of an array, or first key of an object */ +static inline struct flb_json_node *flb_json_node_first(struct flb_json_node *node) +{ + yyjson_val *value; + + value = flb_json_node_val(node); + if ((!yyjson_is_arr(value) && !yyjson_is_obj(value)) || + unsafe_yyjson_get_len(value) == 0) { + return NULL; + } + + return flb_json_node_wrap(unsafe_yyjson_get_first(value)); +} + +/* element or key that follows the given element or object value */ +static inline struct flb_json_node *flb_json_node_next(struct flb_json_node *node) +{ + if (node == NULL) { + return NULL; + } + + /* callers stop on the container size, the last element has no successor */ + return flb_json_node_wrap(unsafe_yyjson_get_next(flb_json_node_val(node))); +} + +/* value of an object key */ +static inline struct flb_json_node *flb_json_node_key_value(struct flb_json_node *key) +{ + if (key == NULL) { + return NULL; + } + + /* in a yyjson tree each object value follows its key */ + return flb_json_node_wrap(flb_json_node_val(key) + 1); +} + +/* value of the first key that matches name ignoring ASCII case, or NULL */ +static inline struct flb_json_node *flb_json_node_obj_get_case(struct flb_json_node *object, + const char *name, + size_t length) +{ + size_t index; + size_t maximum; + yyjson_val *key; + yyjson_val *value; + + if (name == NULL || !yyjson_is_obj(flb_json_node_val(object))) { + return NULL; + } + + yyjson_obj_foreach(flb_json_node_val(object), index, maximum, key, value) { + if (unsafe_yyjson_get_len(key) == length && + strncasecmp(unsafe_yyjson_get_str(key), name, length) == 0) { + return flb_json_node_wrap(value); + } + } + + return NULL; +} + +#else + +/* without a tree backend no tree is read, so no node exists */ +static inline struct flb_json_node *flb_json_tree_root(struct flb_json_tree *tree) +{ + (void) tree; + return NULL; +} + +static inline int flb_json_node_type(struct flb_json_node *node) +{ + (void) node; + return FLB_JSON_NODE_NONE; +} + +static inline int flb_json_node_is_null(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_bool(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_uint(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_sint(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_int(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_real(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_str(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_arr(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_obj(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_get_bool(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline uint64_t flb_json_node_get_uint(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int64_t flb_json_node_get_sint(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline double flb_json_node_get_real(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline const char *flb_json_node_get_str(struct flb_json_node *node) +{ + (void) node; + return NULL; +} + +static inline size_t flb_json_node_get_len(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline const char *flb_json_node_get_strn(struct flb_json_node *node, + size_t *length) +{ + (void) node; + *length = 0; + return NULL; +} + +static inline size_t flb_json_node_arr_size(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline size_t flb_json_node_obj_size(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline struct flb_json_node *flb_json_node_first(struct flb_json_node *node) +{ + (void) node; + return NULL; +} + +static inline struct flb_json_node *flb_json_node_next(struct flb_json_node *node) +{ + (void) node; + return NULL; +} + +static inline struct flb_json_node *flb_json_node_key_value(struct flb_json_node *key) +{ + (void) key; + return NULL; +} + +static inline struct flb_json_node *flb_json_node_obj_get_case(struct flb_json_node *object, + const char *name, + size_t length) +{ + (void) object; + (void) name; + (void) length; + return NULL; +} + +#endif + +#define flb_json_node_arr_foreach(array, index, maximum, value) \ + for ((index) = 0, (maximum) = flb_json_node_arr_size(array), \ + (value) = flb_json_node_first(array); \ + (index) < (maximum); \ + (index)++, (value) = flb_json_node_next(value)) + +#define flb_json_node_obj_foreach(object, index, maximum, key, value) \ + for ((index) = 0, (maximum) = flb_json_node_obj_size(object), \ + (key) = flb_json_node_first(object), \ + (value) = flb_json_node_key_value(key); \ + (index) < (maximum); \ + (index)++, (key) = flb_json_node_next(value), \ + (value) = flb_json_node_key_value(key)) + +#endif diff --git a/src/flb_json.c b/src/flb_json.c index 3e8e1b5cb65..bf5bcc76d4f 100644 --- a/src/flb_json.c +++ b/src/flb_json.c @@ -18,6 +18,7 @@ */ #include +#include #include #include @@ -1877,3 +1878,39 @@ int flb_json_mut_obj_add_val(struct flb_json_mut_doc *document, return flb_json_mut_obj_add_val_len(document, object, key, strlen(key), value); } + +#ifdef FLB_HAVE_YYJSON +struct flb_json_tree *flb_json_tree_read(const char *input, size_t length) +{ + yyjson_alc allocator; + + if (input == NULL) { + return NULL; + } + + flb_json_yyjson_init_alc(&allocator); + + return (struct flb_json_tree *) yyjson_read_opts((char *) input, length, + YYJSON_READ_ALLOW_INVALID_UNICODE | + YYJSON_READ_REPLACE_INVALID_UNICODE, + &allocator, NULL); +} + +void flb_json_tree_destroy(struct flb_json_tree *tree) +{ + yyjson_doc_free((yyjson_doc *) tree); +} +#else +struct flb_json_tree *flb_json_tree_read(const char *input, size_t length) +{ + (void) input; + (void) length; + + return NULL; +} + +void flb_json_tree_destroy(struct flb_json_tree *tree) +{ + (void) tree; +} +#endif From d249bad8637457fda972aa94fee58cdf33fd8fee Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 31 Oct 2026 11:11:45 -0600 Subject: [PATCH 30/33] opentelemetry: read JSON logs through the flb_json tree The JSON logs decoder used yyjson directly, which broke builds with FLB_YYJSON=Off, and its strict parse rejected requests with invalid Unicode that the established conversion accepts with a replacement. Read the request through flb_json_tree, which replaces invalid Unicode the same way. Requests the tree does not read, such as content after the request object or any request in a build without yyjson, are left to the established conversion, so they are handled exactly as before. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_opentelemetry.h | 7 +- .../flb_opentelemetry_logs_json.c | 430 +++++++++--------- 2 files changed, 218 insertions(+), 219 deletions(-) diff --git a/include/fluent-bit/flb_opentelemetry.h b/include/fluent-bit/flb_opentelemetry.h index ff214970839..34bff108765 100644 --- a/include/fluent-bit/flb_opentelemetry.h +++ b/include/fluent-bit/flb_opentelemetry.h @@ -217,9 +217,10 @@ int flb_opentelemetry_logs_json_to_msgpack(struct flb_log_event_encoder *encoder int *error_status); /* - * Returned by flb_opentelemetry_logs_json_to_cfl() for a valid payload whose - * records CFL cannot hold, such as a map key that is not a string. Use - * flb_opentelemetry_logs_json_to_msgpack() for it. + * Returned by flb_opentelemetry_logs_json_to_cfl() for a payload it leaves to + * flb_opentelemetry_logs_json_to_msgpack(): records CFL cannot hold, such as a + * map key that is not a string, and payloads the JSON tree does not read, + * such as content after the request object or a build without yyjson. */ #define FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED -2 diff --git a/src/opentelemetry/flb_opentelemetry_logs_json.c b/src/opentelemetry/flb_opentelemetry_logs_json.c index 04e83567f80..675053507ff 100644 --- a/src/opentelemetry/flb_opentelemetry_logs_json.c +++ b/src/opentelemetry/flb_opentelemetry_logs_json.c @@ -20,13 +20,13 @@ #include #include #include +#include #include #include #include #include #include -#include #include #include @@ -57,11 +57,12 @@ struct json_cfl_builder { struct flb_opentelemetry_logs_json_cfl { struct flb_mp_chunk_cobj *chunk; struct cfl_arena *arena; - yyjson_doc *document; + struct flb_json_tree *document; size_t encoded_size; size_t record_count; }; + static void *json_arena_malloc(void *context, size_t size) { (void) context; @@ -318,37 +319,10 @@ static struct flb_mp_chunk_record *json_cfl_record_add(struct json_cfl_builder * /* * Keys are compared case-insensitively. The expected length is known at * compile time, so most keys are rejected by their length alone; callers - * pass string literals through the json_key_equal() and json_obj_get() - * macros. + * pass string literals through the json_obj_get() macro. */ -static int json_key_equal_n(yyjson_val *key, const char *expected, size_t expected_length) -{ - if (!yyjson_is_str(key) || yyjson_get_len(key) != expected_length) { - return FLB_FALSE; - } - return strncasecmp(yyjson_get_str(key), expected, expected_length) == 0; -} - -static yyjson_val *json_obj_get_n(yyjson_val *object, const char *name, size_t name_length) -{ - size_t index; - size_t maximum; - yyjson_val *key; - yyjson_val *value; - - if (!yyjson_is_obj(object)) { - return NULL; - } - yyjson_obj_foreach(object, index, maximum, key, value) { - if (json_key_equal_n(key, name, name_length)) { - return value; - } - } - return NULL; -} - -#define json_key_equal(key, literal) json_key_equal_n(key, "" literal, sizeof(literal) - 1) -#define json_obj_get(object, literal) json_obj_get_n(object, "" literal, sizeof(literal) - 1) +#define json_obj_get(object, literal) \ + flb_json_node_obj_get_case(object, "" literal, sizeof(literal) - 1) static int json_base64_decode(const char *source, size_t source_size, unsigned char **output, size_t *output_size) @@ -393,9 +367,9 @@ static int json_base64_decode(const char *source, size_t source_size, } static struct cfl_variant *json_plain_value(struct json_cfl_builder *builder, - yyjson_val *value); + struct flb_json_node *value); static struct cfl_variant *json_converted_value(struct json_cfl_builder *builder, - yyjson_val *value, size_t depth); + struct flb_json_node *value, size_t depth); /* * The conversion of OTLP JSON values below mirrors the established msgpack @@ -418,66 +392,72 @@ static struct cfl_variant *json_converted_value(struct json_cfl_builder *builder #define JSON_WRAPPED_KVLIST 7 /* like the established path, a wrapper key matches any prefix of the name */ -static int json_wrapper_key(yyjson_val *key, const char *name) +static int json_wrapper_key(const char *key, size_t key_length, const char *name) { - return strncasecmp(yyjson_get_str(key), name, yyjson_get_len(key)) == 0; + return strncasecmp(key, name, key_length) == 0; } -static int json_wrapped(yyjson_val *object, yyjson_val **out_value, int *out_type) +static int json_wrapped(struct flb_json_node *object, struct flb_json_node **out_value, + int *out_type) { int type; - yyjson_val *key; - yyjson_val *value; - yyjson_obj_iter iterator; + int value_type; + size_t key_length; + const char *key_name; + struct flb_json_node *key; + struct flb_json_node *value; - if (!yyjson_is_obj(object)) { + if (flb_json_node_type(object) != FLB_JSON_NODE_OBJ) { return -1; } type = -1; value = NULL; - if (yyjson_obj_size(object) == 1) { - yyjson_obj_iter_init(object, &iterator); - key = yyjson_obj_iter_next(&iterator); - value = yyjson_obj_iter_get_val(key); - - if (json_wrapper_key(key, "stringValue")) { - if (!yyjson_is_null(value) && !yyjson_is_str(value)) { + value_type = FLB_JSON_NODE_NONE; + if (flb_json_node_obj_size(object) == 1) { + key = flb_json_node_first(object); + value = flb_json_node_key_value(key); + key_name = flb_json_node_get_strn(key, &key_length); + value_type = flb_json_node_type(value); + + if (json_wrapper_key(key_name, key_length, "stringValue")) { + if (value_type != FLB_JSON_NODE_NULL && value_type != FLB_JSON_NODE_STR) { return -2; } type = JSON_WRAPPED_STRING; } - else if (json_wrapper_key(key, "boolValue")) { - if (!yyjson_is_bool(value)) { + else if (json_wrapper_key(key_name, key_length, "boolValue")) { + if (value_type != FLB_JSON_NODE_BOOL) { return -2; } type = JSON_WRAPPED_BOOL; } - else if (json_wrapper_key(key, "intValue")) { - if (!yyjson_is_int(value) && !yyjson_is_str(value)) { + else if (json_wrapper_key(key_name, key_length, "intValue")) { + if (value_type != FLB_JSON_NODE_UINT && value_type != FLB_JSON_NODE_SINT && + value_type != FLB_JSON_NODE_STR) { return -2; } type = JSON_WRAPPED_INT; } - else if (json_wrapper_key(key, "doubleValue")) { - if (!yyjson_is_real(value) && !yyjson_is_str(value)) { + else if (json_wrapper_key(key_name, key_length, "doubleValue")) { + if (value_type != FLB_JSON_NODE_REAL && value_type != FLB_JSON_NODE_STR) { return -2; } type = JSON_WRAPPED_DOUBLE; } - else if (json_wrapper_key(key, "bytesValue")) { - if (!yyjson_is_str(value)) { + else if (json_wrapper_key(key_name, key_length, "bytesValue")) { + if (value_type != FLB_JSON_NODE_STR) { return -2; } type = JSON_WRAPPED_BYTES; } - else if (json_wrapper_key(key, "arrayValue")) { - if (!yyjson_is_arr(value) && !yyjson_is_obj(value)) { + else if (json_wrapper_key(key_name, key_length, "arrayValue")) { + if (value_type != FLB_JSON_NODE_ARR && value_type != FLB_JSON_NODE_OBJ) { return -2; } type = JSON_WRAPPED_ARRAY; } - else if (json_wrapper_key(key, "kvlistValue")) { - if (!yyjson_is_obj(value)) { + else if (json_wrapper_key(key_name, key_length, "kvlistValue")) { + if (value_type != FLB_JSON_NODE_OBJ) { return -2; } type = JSON_WRAPPED_KVLIST; @@ -489,19 +469,20 @@ static int json_wrapped(yyjson_val *object, yyjson_val **out_value, int *out_typ if (out_type != NULL) { *out_type = type; } - if (yyjson_is_obj(value) && yyjson_obj_size(value) == 1) { - yyjson_obj_iter_init(value, &iterator); - key = yyjson_obj_iter_next(&iterator); - if (yyjson_get_len(key) != 6 || strncasecmp(yyjson_get_str(key), "values", 6) != 0) { + if (value_type == FLB_JSON_NODE_OBJ && flb_json_node_obj_size(value) == 1) { + key = flb_json_node_first(value); + if (flb_json_node_get_len(key) != 6 || + strncasecmp(flb_json_node_get_str(key), "values", 6) != 0) { return -3; } - value = yyjson_obj_iter_get_val(key); + value = flb_json_node_key_value(key); } - if (type == JSON_WRAPPED_ARRAY && !yyjson_is_arr(value) && - (!yyjson_is_obj(value) || yyjson_obj_size(value) != 0)) { + if (type == JSON_WRAPPED_ARRAY && !flb_json_node_is_arr(value) && + (!flb_json_node_is_obj(value) || flb_json_node_obj_size(value) != 0)) { return -2; } - else if (type == JSON_WRAPPED_KVLIST && !yyjson_is_arr(value) && !yyjson_is_obj(value)) { + else if (type == JSON_WRAPPED_KVLIST && !flb_json_node_is_arr(value) && + !flb_json_node_is_obj(value)) { return -2; } if (out_value != NULL) { @@ -511,11 +492,11 @@ static int json_wrapped(yyjson_val *object, yyjson_val **out_value, int *out_typ } static struct cfl_variant *json_converted_array(struct json_cfl_builder *builder, - yyjson_val *array, size_t depth) + struct flb_json_node *array, size_t depth) { size_t index; size_t maximum; - yyjson_val *entry; + struct flb_json_node *entry; struct cfl_array *result; struct cfl_variant *variant; @@ -526,11 +507,11 @@ static struct cfl_variant *json_converted_array(struct json_cfl_builder *builder if (json_cfl_account(builder, OTEL_JSON_CFL_ARRAY_HEADER_SIZE) != 0) { return NULL; } - result = cfl_array_create_in(builder->arena, yyjson_arr_size(array)); + result = cfl_array_create_in(builder->arena, flb_json_node_arr_size(array)); if (result == NULL) { return NULL; } - yyjson_arr_foreach(array, index, maximum, entry) { + flb_json_node_arr_foreach(array, index, maximum, entry) { variant = json_converted_value(builder, entry, depth + 1); if (variant == NULL || cfl_array_append(result, variant) != 0) { if (variant != NULL) { @@ -564,7 +545,7 @@ static struct cfl_variant *json_kvlist_variant(struct json_cfl_builder *builder, /* Insert a converted key, CFL only holds string keys */ static int json_converted_insert(struct json_cfl_builder *builder, struct cfl_kvlist *kvlist, - yyjson_val *key, struct cfl_variant *value, size_t depth) + struct flb_json_node *key, struct cfl_variant *value, size_t depth) { int result; size_t size; @@ -573,9 +554,9 @@ static int json_converted_insert(struct json_cfl_builder *builder, struct cfl_kv if (value == NULL) { return -1; } - if (yyjson_is_str(key)) { - return json_cfl_insert(builder, kvlist, yyjson_get_str(key), yyjson_get_len(key), - value); + if (flb_json_node_is_str(key)) { + return json_cfl_insert(builder, kvlist, flb_json_node_get_str(key), + flb_json_node_get_len(key), value); } /* the key is accounted once, by the insertion */ @@ -597,17 +578,17 @@ static int json_converted_insert(struct json_cfl_builder *builder, struct cfl_kv } static struct cfl_kvlist *json_converted_kvlist(struct json_cfl_builder *builder, - yyjson_val *array, size_t depth) + struct flb_json_node *array, size_t depth) { size_t index; size_t maximum; - yyjson_val *entry; - yyjson_val *key; - yyjson_val *value; + struct flb_json_node *entry; + struct flb_json_node *key; + struct flb_json_node *value; struct cfl_kvlist *result; struct cfl_variant *variant; - if (!yyjson_is_arr(array)) { + if (!flb_json_node_is_arr(array)) { return NULL; } if (depth > OTEL_JSON_CFL_MAX_DEPTH) { @@ -618,9 +599,9 @@ static struct cfl_kvlist *json_converted_kvlist(struct json_cfl_builder *builder if (result == NULL) { return NULL; } - yyjson_arr_foreach(array, index, maximum, entry) { + flb_json_node_arr_foreach(array, index, maximum, entry) { key = json_obj_get(entry, "key"); - if (!yyjson_is_obj(entry) || key == NULL) { + if (!flb_json_node_is_obj(entry) || key == NULL) { cfl_kvlist_destroy(result); return NULL; } @@ -629,10 +610,10 @@ static struct cfl_kvlist *json_converted_kvlist(struct json_cfl_builder *builder /* an unset value is an empty string */ variant = json_cfl_string(builder, "", 0); } - else if (yyjson_is_obj(value) && yyjson_obj_size(value) == 0) { + else if (flb_json_node_is_obj(value) && flb_json_node_obj_size(value) == 0) { variant = json_cfl_null(builder); } - else if (!yyjson_is_obj(value)) { + else if (!flb_json_node_is_obj(value)) { cfl_kvlist_destroy(result); return NULL; } @@ -648,13 +629,13 @@ static struct cfl_kvlist *json_converted_kvlist(struct json_cfl_builder *builder } static struct cfl_variant *json_converted_map(struct json_cfl_builder *builder, - yyjson_val *object, size_t depth); + struct flb_json_node *object, size_t depth); /* * Returns 0 with the converted value, NULL on failure, or the json_wrapped() * status when the object is not a valid wrapper. */ -static int json_unwrapped_value(struct json_cfl_builder *builder, yyjson_val *object, +static int json_unwrapped_value(struct json_cfl_builder *builder, struct flb_json_node *object, size_t depth, struct cfl_variant **out_variant) { int ret; @@ -663,7 +644,7 @@ static int json_unwrapped_value(struct json_cfl_builder *builder, yyjson_val *ob size_t bytes_size; unsigned char *bytes; char buffer[33]; - yyjson_val *value; + struct flb_json_node *value; struct cfl_array *array; ret = json_wrapped(object, &value, &type); @@ -674,19 +655,19 @@ static int json_unwrapped_value(struct json_cfl_builder *builder, yyjson_val *ob switch (type) { case JSON_WRAPPED_INT: - if (yyjson_is_str(value)) { - length = yyjson_get_len(value); + if (flb_json_node_is_str(value)) { + length = flb_json_node_get_len(value); if (length > sizeof(buffer) - 1) { length = sizeof(buffer) - 1; } - memcpy(buffer, yyjson_get_str(value), length); + memcpy(buffer, flb_json_node_get_str(value), length); buffer[length] = '\0'; *out_variant = json_cfl_int(builder, strtoll(buffer, NULL, 10)); return 0; } break; case JSON_WRAPPED_BYTES: - if (json_base64_decode(yyjson_get_str(value), yyjson_get_len(value), + if (json_base64_decode(flb_json_node_get_str(value), flb_json_node_get_len(value), &bytes, &bytes_size) != 0) { return -2; } @@ -698,7 +679,7 @@ static int json_unwrapped_value(struct json_cfl_builder *builder, yyjson_val *ob flb_free(bytes); return 0; case JSON_WRAPPED_ARRAY: - if (yyjson_is_arr(value)) { + if (flb_json_node_is_arr(value)) { *out_variant = json_converted_array(builder, value, depth + 1); return 0; } @@ -714,7 +695,7 @@ static int json_unwrapped_value(struct json_cfl_builder *builder, yyjson_val *ob } return 0; case JSON_WRAPPED_KVLIST: - if (yyjson_is_arr(value)) { + if (flb_json_node_is_arr(value)) { *out_variant = json_kvlist_variant(builder, json_converted_kvlist(builder, value, depth + 1)); @@ -733,13 +714,13 @@ static int json_unwrapped_value(struct json_cfl_builder *builder, yyjson_val *ob } static struct cfl_variant *json_converted_map(struct json_cfl_builder *builder, - yyjson_val *object, size_t depth) + struct flb_json_node *object, size_t depth) { int ret; size_t index; size_t maximum; - yyjson_val *key; - yyjson_val *value; + struct flb_json_node *key; + struct flb_json_node *value; struct cfl_kvlist *kvlist; struct cfl_variant *variant; @@ -760,7 +741,7 @@ static struct cfl_variant *json_converted_map(struct json_cfl_builder *builder, if (kvlist == NULL) { return NULL; } - yyjson_obj_foreach(object, index, maximum, key, value) { + flb_json_node_obj_foreach(object, index, maximum, key, value) { if (json_converted_insert(builder, kvlist, key, json_converted_value(builder, value, depth + 1), depth) != 0) { @@ -772,29 +753,30 @@ static struct cfl_variant *json_converted_map(struct json_cfl_builder *builder, } static struct cfl_variant *json_converted_value(struct json_cfl_builder *builder, - yyjson_val *value, size_t depth) + struct flb_json_node *value, size_t depth) { - if (yyjson_is_obj(value)) { + switch (flb_json_node_type(value)) { + case FLB_JSON_NODE_OBJ: return json_converted_map(builder, value, depth + 1); - } - if (yyjson_is_arr(value)) { + case FLB_JSON_NODE_ARR: return json_converted_array(builder, value, depth + 1); - } - if (yyjson_is_null(value)) { + case FLB_JSON_NODE_NULL: /* the established path encodes JSON null as an empty string */ return json_cfl_string(builder, "", 0); + default: + return json_plain_value(builder, value); } - return json_plain_value(builder, value); } /* Copy a JSON value as is, the way the established path copies msgpack objects */ static struct cfl_variant *json_raw_value(struct json_cfl_builder *builder, - yyjson_val *value, size_t depth) + struct flb_json_node *value, size_t depth) { + int type; size_t index; size_t maximum; - yyjson_val *key; - yyjson_val *entry; + struct flb_json_node *key; + struct flb_json_node *entry; struct cfl_array *array; struct cfl_kvlist *kvlist; struct cfl_variant *variant; @@ -802,17 +784,19 @@ static struct cfl_variant *json_raw_value(struct json_cfl_builder *builder, if (depth > OTEL_JSON_CFL_MAX_DEPTH) { return NULL; } - if (yyjson_is_obj(value)) { + type = flb_json_node_type(value); + if (type == FLB_JSON_NODE_OBJ) { if (json_cfl_account(builder, - json_cfl_array_header_size(yyjson_obj_size(value))) != 0) { + json_cfl_array_header_size(flb_json_node_obj_size(value))) != 0) { return NULL; } kvlist = cfl_kvlist_create_in(builder->arena); if (kvlist == NULL) { return NULL; } - yyjson_obj_foreach(value, index, maximum, key, entry) { - if (json_cfl_insert(builder, kvlist, yyjson_get_str(key), yyjson_get_len(key), + flb_json_node_obj_foreach(value, index, maximum, key, entry) { + if (json_cfl_insert(builder, kvlist, flb_json_node_get_str(key), + flb_json_node_get_len(key), json_raw_value(builder, entry, depth + 1)) != 0) { cfl_kvlist_destroy(kvlist); return NULL; @@ -824,16 +808,16 @@ static struct cfl_variant *json_raw_value(struct json_cfl_builder *builder, } return variant; } - if (yyjson_is_arr(value)) { + if (type == FLB_JSON_NODE_ARR) { if (json_cfl_account(builder, - json_cfl_array_header_size(yyjson_arr_size(value))) != 0) { + json_cfl_array_header_size(flb_json_node_arr_size(value))) != 0) { return NULL; } - array = cfl_array_create_in(builder->arena, yyjson_arr_size(value)); + array = cfl_array_create_in(builder->arena, flb_json_node_arr_size(value)); if (array == NULL) { return NULL; } - yyjson_arr_foreach(value, index, maximum, entry) { + flb_json_node_arr_foreach(value, index, maximum, entry) { variant = json_raw_value(builder, entry, depth + 1); if (variant == NULL || cfl_array_append(array, variant) != 0) { if (variant != NULL) { @@ -857,18 +841,18 @@ static struct cfl_variant *json_raw_value(struct json_cfl_builder *builder, * characters are considered, and they must all be digits. Values beyond * 64 bits saturate. Returns the number of digits, or -1. */ -static int json_parse_digits(yyjson_val *value, uint64_t *result) +static int json_parse_digits(struct flb_json_node *value, uint64_t *result) { size_t index; size_t length; const char *string; char buffer[32]; - if (!yyjson_is_str(value)) { + if (!flb_json_node_is_str(value)) { return -1; } - string = yyjson_get_str(value); - length = yyjson_get_len(value); + string = flb_json_node_get_str(value); + length = flb_json_node_get_len(value); if (length > sizeof(buffer) - 1) { length = sizeof(buffer) - 1; } @@ -884,12 +868,12 @@ static int json_parse_digits(yyjson_val *value, uint64_t *result) } /* The timestamp of a log record, it must be an unsigned integer or digits */ -static int json_record_timestamp(yyjson_val *value, struct flb_time *tm) +static int json_record_timestamp(struct flb_json_node *value, struct flb_time *tm) { uint64_t number; - if (yyjson_is_uint(value)) { - number = yyjson_get_uint(value); + if (flb_json_node_is_uint(value)) { + number = flb_json_node_get_uint(value); } else if (json_parse_digits(value, &number) < 0) { return -1; @@ -898,21 +882,21 @@ static int json_record_timestamp(yyjson_val *value, struct flb_time *tm) } /* A non-empty identifier must have the exact hexadecimal length */ -static int json_identifier_valid(yyjson_val *value, size_t length) +static int json_identifier_valid(struct flb_json_node *value, size_t length) { size_t index; const char *string; - if (!yyjson_is_str(value)) { + if (!flb_json_node_is_str(value)) { return FLB_FALSE; } - if (yyjson_get_len(value) == 0) { + if (flb_json_node_get_len(value) == 0) { return FLB_TRUE; } - if (yyjson_get_len(value) != length) { + if (flb_json_node_get_len(value) != length) { return FLB_FALSE; } - string = yyjson_get_str(value); + string = flb_json_node_get_str(value); for (index = 0; index < length; index++) { if (!isxdigit((unsigned char) string[index])) { return FLB_FALSE; @@ -925,12 +909,12 @@ static int json_identifier_valid(yyjson_val *value, size_t length) * Reject a log record the established path rejects, with the same status and * in the same order, before any of it is built. */ -static int json_record_validate(struct json_cfl_builder *builder, yyjson_val *log_record, +static int json_record_validate(struct json_cfl_builder *builder, struct flb_json_node *log_record, struct flb_time *tm) { - yyjson_val *value; + struct flb_json_node *value; - if (!yyjson_is_obj(log_record)) { + if (!flb_json_node_is_obj(log_record)) { return FLB_OTEL_LOGS_ERR_UNEXPECTED_LOGRECORDS_ENTRY_TYPE; } value = json_obj_get(log_record, "timeUnixNano"); @@ -949,7 +933,7 @@ static int json_record_validate(struct json_cfl_builder *builder, yyjson_val *lo return FLB_OTEL_LOGS_ERR_UNEXPECTED_TIMESTAMP_TYPE; } value = json_obj_get(log_record, "attributes"); - if (value != NULL && !yyjson_is_arr(value)) { + if (value != NULL && !flb_json_node_is_arr(value)) { return FLB_OTEL_LOGS_ERR_UNEXPECTED_ATTRIBUTES_TYPE; } value = json_obj_get(log_record, "traceId"); @@ -961,43 +945,44 @@ static int json_record_validate(struct json_cfl_builder *builder, yyjson_val *lo return FLB_OTEL_LOGS_ERR_INVALID_SPAN_ID; } value = json_obj_get(log_record, "body"); - if (value != NULL && !yyjson_is_obj(value)) { + if (value != NULL && !flb_json_node_is_obj(value)) { return FLB_OTEL_LOGS_ERR_UNEXPECTED_BODY_TYPE; } return 0; } static struct cfl_variant *json_plain_value(struct json_cfl_builder *builder, - yyjson_val *value) + struct flb_json_node *value) { - if (yyjson_is_str(value)) { - return json_cfl_string(builder, yyjson_get_str(value), yyjson_get_len(value)); - } - if (yyjson_is_uint(value)) { - return json_cfl_uint(builder, yyjson_get_uint(value)); - } - if (yyjson_is_sint(value)) { - return json_cfl_int(builder, yyjson_get_sint(value)); - } - if (yyjson_is_real(value)) { - return json_cfl_double(builder, yyjson_get_real(value)); - } - if (yyjson_is_bool(value)) { - return json_cfl_bool(builder, yyjson_get_bool(value)); - } - if (yyjson_is_null(value)) { + size_t length; + const char *string; + + switch (flb_json_node_type(value)) { + case FLB_JSON_NODE_STR: + string = flb_json_node_get_strn(value, &length); + return json_cfl_string(builder, string, length); + case FLB_JSON_NODE_UINT: + return json_cfl_uint(builder, flb_json_node_get_uint(value)); + case FLB_JSON_NODE_SINT: + return json_cfl_int(builder, flb_json_node_get_sint(value)); + case FLB_JSON_NODE_REAL: + return json_cfl_double(builder, flb_json_node_get_real(value)); + case FLB_JSON_NODE_BOOL: + return json_cfl_bool(builder, flb_json_node_get_bool(value)); + case FLB_JSON_NODE_NULL: return json_cfl_null(builder); + default: + return NULL; } - return NULL; } static struct cfl_kvlist *json_record_metadata(struct json_cfl_builder *builder, - yyjson_val *log_record) + struct flb_json_node *log_record) { int result; uint64_t number; unsigned char identifier[16]; - yyjson_val *value; + struct flb_json_node *value; struct cfl_kvlist *metadata; struct cfl_kvlist *otlp; @@ -1019,10 +1004,10 @@ static struct cfl_kvlist *json_record_metadata(struct json_cfl_builder *builder, json_raw_value(builder, value, 0)); } value = json_obj_get(log_record, "severityText"); - if (result == 0 && yyjson_is_str(value)) { + if (result == 0 && flb_json_node_is_str(value)) { result = json_cfl_insert(builder, otlp, "severity_text", 13, - json_cfl_string(builder, yyjson_get_str(value), - yyjson_get_len(value))); + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); } value = json_obj_get(log_record, "attributes"); if (result == 0 && value != NULL) { @@ -1033,14 +1018,15 @@ static struct cfl_kvlist *json_record_metadata(struct json_cfl_builder *builder, } } value = json_obj_get(log_record, "droppedAttributesCount"); - if (result == 0 && value != NULL && (yyjson_is_int(value) || yyjson_is_uint(value))) { + if (result == 0 && value != NULL && + (flb_json_node_is_int(value) || flb_json_node_is_uint(value))) { result = json_cfl_insert(builder, otlp, "dropped_attributes_count", 24, json_plain_value(builder, value)); } value = json_obj_get(log_record, "traceId"); - if (result == 0 && yyjson_is_str(value) && yyjson_get_len(value) > 0) { - if (yyjson_get_len(value) != 32 || - flb_otel_utils_hex_to_id(yyjson_get_str(value), 32, identifier, 16) != 0 || + if (result == 0 && flb_json_node_is_str(value) && flb_json_node_get_len(value) > 0) { + if (flb_json_node_get_len(value) != 32 || + flb_otel_utils_hex_to_id(flb_json_node_get_str(value), 32, identifier, 16) != 0 || json_cfl_account_sized(builder, json_cfl_binary_header_size(16), 16) != 0) { goto error; } @@ -1049,9 +1035,9 @@ static struct cfl_kvlist *json_record_metadata(struct json_cfl_builder *builder, (char *) identifier, 16, CFL_FALSE)); } value = json_obj_get(log_record, "spanId"); - if (result == 0 && yyjson_is_str(value) && yyjson_get_len(value) > 0) { - if (yyjson_get_len(value) != 16 || - flb_otel_utils_hex_to_id(yyjson_get_str(value), 16, identifier, 8) != 0 || + if (result == 0 && flb_json_node_is_str(value) && flb_json_node_get_len(value) > 0) { + if (flb_json_node_get_len(value) != 16 || + flb_otel_utils_hex_to_id(flb_json_node_get_str(value), 16, identifier, 8) != 0 || json_cfl_account_sized(builder, json_cfl_binary_header_size(8), 8) != 0) { goto error; } @@ -1063,18 +1049,18 @@ static struct cfl_kvlist *json_record_metadata(struct json_cfl_builder *builder, if (value == NULL) { value = json_obj_get(log_record, "traceFlags"); } - if (result == 0 && (yyjson_is_uint(value) || json_parse_digits(value, &number) > 0)) { - if (yyjson_is_uint(value)) { - number = yyjson_get_uint(value); + if (result == 0 && (flb_json_node_is_uint(value) || json_parse_digits(value, &number) > 0)) { + if (flb_json_node_is_uint(value)) { + number = flb_json_node_get_uint(value); } result = json_cfl_insert(builder, otlp, "trace_flags", 11, json_cfl_uint(builder, number & 0xff)); } value = json_obj_get(log_record, "eventName"); - if (result == 0 && yyjson_is_str(value)) { + if (result == 0 && flb_json_node_is_str(value)) { result = json_cfl_insert(builder, otlp, "event_name", 10, - json_cfl_string(builder, yyjson_get_str(value), - yyjson_get_len(value))); + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); } if (result == 0) { result = json_cfl_insert_kvlist(builder, metadata, "otlp", 4, otlp); @@ -1091,11 +1077,11 @@ static struct cfl_kvlist *json_record_metadata(struct json_cfl_builder *builder, } static struct cfl_kvlist *json_record_body(struct json_cfl_builder *builder, - yyjson_val *log_record) + struct flb_json_node *log_record) { int type; const char *key; - yyjson_val *body; + struct flb_json_node *body; struct cfl_kvlist *result; struct cfl_variant *variant; @@ -1160,14 +1146,14 @@ static struct cfl_kvlist *json_group_metadata(struct json_cfl_builder *builder, } static struct cfl_kvlist *json_group_body(struct json_cfl_builder *builder, - yyjson_val *resource_log, - yyjson_val *scope_log) + struct flb_json_node *resource_log, + struct flb_json_node *scope_log) { int result; - yyjson_val *resource; - yyjson_val *scope; - yyjson_val *schema_url; - yyjson_val *value; + struct flb_json_node *resource; + struct flb_json_node *scope; + struct flb_json_node *schema_url; + struct flb_json_node *value; struct cfl_kvlist *body; struct cfl_kvlist *resource_map; struct cfl_kvlist *scope_map; @@ -1180,7 +1166,7 @@ static struct cfl_kvlist *json_group_body(struct json_cfl_builder *builder, goto error; } resource = json_obj_get(resource_log, "resource"); - if (yyjson_is_obj(resource)) { + if (flb_json_node_is_obj(resource)) { value = json_obj_get(resource, "attributes"); if (value != NULL) { result = json_cfl_insert_kvlist(builder, resource_map, "attributes", 10, @@ -1196,21 +1182,21 @@ static struct cfl_kvlist *json_group_body(struct json_cfl_builder *builder, } } value = json_obj_get(resource_log, "schemaUrl"); - if (result == 0 && yyjson_is_str(value)) { + if (result == 0 && flb_json_node_is_str(value)) { result = json_cfl_insert(builder, resource_map, "schema_url", 10, - json_cfl_string(builder, yyjson_get_str(value), - yyjson_get_len(value))); + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); } if (result == 0) { result = json_cfl_insert_kvlist(builder, body, "resource", 8, resource_map); resource_map = NULL; } scope = json_obj_get(scope_log, "scope"); - if (!yyjson_is_obj(scope)) { + if (!flb_json_node_is_obj(scope)) { scope = NULL; } schema_url = json_obj_get(scope_log, "schemaUrl"); - if (!yyjson_is_str(schema_url)) { + if (!flb_json_node_is_str(schema_url)) { schema_url = NULL; } @@ -1229,16 +1215,16 @@ static struct cfl_kvlist *json_group_body(struct json_cfl_builder *builder, } if (result == 0 && scope != NULL) { value = json_obj_get(scope, "name"); - if (yyjson_is_str(value)) { + if (flb_json_node_is_str(value)) { result = json_cfl_insert(builder, scope_map, "name", 4, - json_cfl_string(builder, yyjson_get_str(value), - yyjson_get_len(value))); + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); } value = json_obj_get(scope, "version"); - if (result == 0 && yyjson_is_str(value)) { + if (result == 0 && flb_json_node_is_str(value)) { result = json_cfl_insert(builder, scope_map, "version", 7, - json_cfl_string(builder, yyjson_get_str(value), - yyjson_get_len(value))); + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); } value = json_obj_get(scope, "droppedAttributesCount"); if (result == 0 && value != NULL) { @@ -1256,8 +1242,8 @@ static struct cfl_kvlist *json_group_body(struct json_cfl_builder *builder, } if (result == 0 && schema_url != NULL) { result = json_cfl_insert(builder, scope_map, "schema_url", 10, - json_cfl_string(builder, yyjson_get_str(schema_url), - yyjson_get_len(schema_url))); + json_cfl_string(builder, flb_json_node_get_str(schema_url), + flb_json_node_get_len(schema_url))); } if (result == 0) { result = json_cfl_insert_kvlist(builder, body, "scope", 5, scope_map); @@ -1275,20 +1261,21 @@ static struct cfl_kvlist *json_group_body(struct json_cfl_builder *builder, } static int json_scope_logs(struct json_cfl_builder *builder, - yyjson_val *resource_log, yyjson_val *scope_log, + struct flb_json_node *resource_log, struct flb_json_node *scope_log, size_t resource_index, size_t scope_index, size_t *record_count) { size_t index; size_t maximum; - yyjson_val *records; - yyjson_val *log_record; + struct flb_json_node *records; + struct flb_json_node *log_record; struct flb_time tm; struct flb_mp_chunk_record *group; struct flb_mp_chunk_record *record; records = json_obj_get(scope_log, "logRecords"); - if (records == NULL || (yyjson_is_arr(records) && yyjson_arr_size(records) == 0)) { + if (records == NULL || + (flb_json_node_is_arr(records) && flb_json_node_arr_size(records) == 0)) { return 0; } flb_time_set(&tm, FLB_LOG_EVENT_GROUP_START, 0); @@ -1303,7 +1290,7 @@ static int json_scope_logs(struct json_cfl_builder *builder, if (group->cobj_metadata == NULL || group->cobj_record == NULL) { return -1; } - if (!yyjson_is_arr(records)) { + if (!flb_json_node_is_arr(records)) { builder->error_status = FLB_OTEL_LOGS_ERR_UNEXPECTED_LOGRECORDS_TYPE; return -1; } @@ -1312,7 +1299,7 @@ static int json_scope_logs(struct json_cfl_builder *builder, group->owns_group_metadata = FLB_TRUE; group->owns_group_attributes = FLB_TRUE; - yyjson_arr_foreach(records, index, maximum, log_record) { + flb_json_node_arr_foreach(records, index, maximum, log_record) { builder->error_status = json_record_validate(builder, log_record, &tm); if (builder->error_status != 0) { return -1; @@ -1363,12 +1350,12 @@ int flb_opentelemetry_logs_json_to_cfl( size_t resource_maximum; size_t scope_index; size_t scope_maximum; - yyjson_val *root; - yyjson_val *resource_logs; - yyjson_val *resource_log; - yyjson_val *scope_logs; - yyjson_val *scope_log; - yyjson_val *resource; + struct flb_json_node *root; + struct flb_json_node *resource_logs; + struct flb_json_node *resource_log; + struct flb_json_node *scope_logs; + struct flb_json_node *scope_log; + struct flb_json_node *resource; struct json_cfl_builder builder; struct flb_opentelemetry_logs_json_cfl *output; @@ -1389,13 +1376,24 @@ int flb_opentelemetry_logs_json_to_cfl( return -1; } output->arena = json_arena_create(); - output->document = yyjson_read(body, length, 0); - if (output->arena == NULL || output->document == NULL) { + if (output->arena == NULL) { result = -1; goto error; } - root = yyjson_doc_get_root(output->document); - if (!yyjson_is_obj(root)) { + + /* + * The tree replaces invalid Unicode like the established conversion does. + * Leave every payload it does not read, such as content after the request + * object or a build without a tree backend, to the established conversion + * so those requests are handled exactly as before. + */ + output->document = flb_json_tree_read(body, length); + if (output->document == NULL) { + result = FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED; + goto error; + } + root = flb_json_tree_root(output->document); + if (!flb_json_node_is_obj(root)) { status = FLB_OTEL_LOGS_ERR_UNEXPECTED_ROOT_OBJECT_TYPE; result = -1; goto error; @@ -1406,7 +1404,7 @@ int flb_opentelemetry_logs_json_to_cfl( result = -1; goto error; } - if (!yyjson_is_arr(resource_logs)) { + if (!flb_json_node_is_arr(resource_logs)) { status = FLB_OTEL_LOGS_ERR_UNEXPECTED_RESOURCELOGS_TYPE; result = -1; goto error; @@ -1424,15 +1422,15 @@ int flb_opentelemetry_logs_json_to_cfl( output->chunk = builder.chunk; /* the checks and their order follow flb_opentelemetry_logs_json_to_msgpack() */ - yyjson_arr_foreach(resource_logs, resource_index, resource_maximum, resource_log) { - if (!yyjson_is_obj(resource_log)) { + flb_json_node_arr_foreach(resource_logs, resource_index, resource_maximum, resource_log) { + if (!flb_json_node_is_obj(resource_log)) { status = FLB_OTEL_LOGS_ERR_UNEXPECTED_RESOURCELOGS_ENTRY_TYPE; result = -1; goto error; } resource = json_obj_get(resource_log, "resource"); - if (yyjson_is_obj(resource) && json_obj_get(resource, "attributes") != NULL && - !yyjson_is_arr(json_obj_get(resource, "attributes"))) { + if (flb_json_node_is_obj(resource) && json_obj_get(resource, "attributes") != NULL && + !flb_json_node_is_arr(json_obj_get(resource, "attributes"))) { status = FLB_OTEL_LOGS_ERR_UNEXPECTED_ATTRIBUTES_TYPE; result = -1; goto error; @@ -1443,13 +1441,13 @@ int flb_opentelemetry_logs_json_to_cfl( result = -1; goto error; } - if (!yyjson_is_arr(scope_logs)) { + if (!flb_json_node_is_arr(scope_logs)) { status = FLB_OTEL_LOGS_ERR_UNEXPECTED_SCOPELOGS_TYPE; result = -1; goto error; } - yyjson_arr_foreach(scope_logs, scope_index, scope_maximum, scope_log) { - if (!yyjson_is_obj(scope_log)) { + flb_json_node_arr_foreach(scope_logs, scope_index, scope_maximum, scope_log) { + if (!flb_json_node_is_obj(scope_log)) { status = FLB_OTEL_LOGS_ERR_UNEXPECTED_SCOPELOGS_TYPE; result = -1; goto error; @@ -1506,6 +1504,6 @@ void flb_opentelemetry_logs_json_cfl_destroy( } flb_mp_chunk_cobj_destroy(result->chunk); cfl_arena_destroy(result->arena); - yyjson_doc_free(result->document); + flb_json_tree_destroy(result->document); flb_free(result); } From fdbebef1870fcf24fb06bd651faec72d99c7fbd0 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 31 Oct 2026 11:11:45 -0600 Subject: [PATCH 31/33] tests: internal: cover the JSON tree accessors Check value types, nested iteration, invalid Unicode replacement, rejected inputs and NULL nodes, and that no tree is read without yyjson. Signed-off-by: Eduardo Silva --- tests/internal/CMakeLists.txt | 1 + tests/internal/json.c | 212 ++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 tests/internal/json.c diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 0b358e9bf2b..4e60f79673c 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -46,6 +46,7 @@ set(UNIT_TESTS_FILES ring_buffer.c regex.c scheduler.c + json.c parser_json.c parser_ltsv.c parser_regex.c diff --git a/tests/internal/json.c b/tests/internal/json.c new file mode 100644 index 00000000000..317dec4c9ae --- /dev/null +++ b/tests/internal/json.c @@ -0,0 +1,212 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#include "flb_tests_internal.h" + +#ifdef FLB_HAVE_YYJSON +static struct flb_json_tree *tree_read(const char *json) +{ + return flb_json_tree_read(json, strlen(json)); +} + +static void test_tree_types() +{ + size_t index; + size_t maximum; + struct flb_json_tree *tree; + struct flb_json_node *root; + struct flb_json_node *key; + struct flb_json_node *value; + static const char *names[] = {"n", "b", "u", "s", "r", "t", "a", "o"}; + + tree = tree_read("{\"n\":null,\"b\":true,\"u\":18446744073709551615," + "\"s\":-42,\"r\":1.5,\"t\":\"text\",\"a\":[1,2],\"o\":{}}"); + TEST_CHECK(tree != NULL); + if (tree == NULL) { + return; + } + + root = flb_json_tree_root(tree); + TEST_CHECK(flb_json_node_is_obj(root)); + TEST_CHECK(flb_json_node_obj_size(root) == 8); + + flb_json_node_obj_foreach(root, index, maximum, key, value) { + TEST_CHECK(flb_json_node_is_str(key)); + TEST_CHECK(flb_json_node_get_len(key) == 1); + TEST_CHECK(strncmp(flb_json_node_get_str(key), names[index], 1) == 0); + + switch (index) { + case 0: + TEST_CHECK(flb_json_node_is_null(value)); + break; + case 1: + TEST_CHECK(flb_json_node_is_bool(value) && flb_json_node_get_bool(value)); + break; + case 2: + TEST_CHECK(flb_json_node_is_uint(value) && flb_json_node_is_int(value)); + TEST_CHECK(flb_json_node_get_uint(value) == UINT64_MAX); + break; + case 3: + TEST_CHECK(flb_json_node_is_sint(value) && flb_json_node_is_int(value)); + TEST_CHECK(flb_json_node_get_sint(value) == -42); + break; + case 4: + TEST_CHECK(flb_json_node_is_real(value) && !flb_json_node_is_int(value)); + TEST_CHECK(flb_json_node_get_real(value) == 1.5); + break; + case 5: + TEST_CHECK(flb_json_node_is_str(value)); + TEST_CHECK(flb_json_node_get_len(value) == 4); + TEST_CHECK(strncmp(flb_json_node_get_str(value), "text", 4) == 0); + break; + case 6: + TEST_CHECK(flb_json_node_is_arr(value)); + TEST_CHECK(flb_json_node_arr_size(value) == 2); + break; + case 7: + TEST_CHECK(flb_json_node_is_obj(value)); + TEST_CHECK(flb_json_node_obj_size(value) == 0); + TEST_CHECK(flb_json_node_first(value) == NULL); + break; + } + } + TEST_CHECK(index == 8); + + flb_json_tree_destroy(tree); +} + +static void test_tree_nested_iteration() +{ + size_t total; + size_t index; + size_t maximum; + size_t inner_index; + size_t inner_maximum; + struct flb_json_tree *tree; + struct flb_json_node *root; + struct flb_json_node *entry; + struct flb_json_node *inner; + + /* containers inside arrays must be skipped as a whole */ + tree = tree_read("[[1,[2,3]],{\"k\":[4,{\"x\":5}]},6]"); + TEST_CHECK(tree != NULL); + if (tree == NULL) { + return; + } + + root = flb_json_tree_root(tree); + TEST_CHECK(flb_json_node_arr_size(root) == 3); + + total = 0; + flb_json_node_arr_foreach(root, index, maximum, entry) { + if (index == 0) { + TEST_CHECK(flb_json_node_is_arr(entry)); + flb_json_node_arr_foreach(entry, inner_index, inner_maximum, inner) { + total++; + } + } + else if (index == 1) { + TEST_CHECK(flb_json_node_is_obj(entry)); + inner = flb_json_node_key_value(flb_json_node_first(entry)); + TEST_CHECK(flb_json_node_is_arr(inner)); + TEST_CHECK(flb_json_node_arr_size(inner) == 2); + } + else { + TEST_CHECK(flb_json_node_is_uint(entry)); + TEST_CHECK(flb_json_node_get_uint(entry) == 6); + } + } + TEST_CHECK(index == 3); + TEST_CHECK(total == 2); + + flb_json_tree_destroy(tree); +} + +static void test_tree_invalid_unicode() +{ + struct flb_json_tree *tree; + struct flb_json_node *root; + + /* an unpaired surrogate is replaced with U+FFFD, like flb_pack_json() */ + tree = tree_read("\"a\\uD800b\""); + TEST_CHECK(tree != NULL); + if (tree == NULL) { + return; + } + + root = flb_json_tree_root(tree); + TEST_CHECK(flb_json_node_is_str(root)); + TEST_CHECK(flb_json_node_get_len(root) == 5); + TEST_CHECK(memcmp(flb_json_node_get_str(root), "a\xef\xbf\xbd" "b", 5) == 0); + + flb_json_tree_destroy(tree); +} + +static void test_tree_rejects() +{ + TEST_CHECK(flb_json_tree_read(NULL, 0) == NULL); + TEST_CHECK(tree_read("") == NULL); + TEST_CHECK(tree_read("{\"a\":") == NULL); + TEST_CHECK(tree_read("{} trailing") == NULL); + TEST_CHECK(tree_read("{}{}") == NULL); +} +#endif + +static void test_tree_null_nodes() +{ + TEST_CHECK(flb_json_tree_root(NULL) == NULL); + TEST_CHECK(!flb_json_node_is_null(NULL)); + TEST_CHECK(!flb_json_node_is_str(NULL)); + TEST_CHECK(!flb_json_node_is_obj(NULL)); + TEST_CHECK(!flb_json_node_is_arr(NULL)); + TEST_CHECK(flb_json_node_get_str(NULL) == NULL); + TEST_CHECK(flb_json_node_get_len(NULL) == 0); + TEST_CHECK(flb_json_node_arr_size(NULL) == 0); + TEST_CHECK(flb_json_node_obj_size(NULL) == 0); + TEST_CHECK(flb_json_node_first(NULL) == NULL); + TEST_CHECK(flb_json_node_next(NULL) == NULL); + TEST_CHECK(flb_json_node_key_value(NULL) == NULL); + flb_json_tree_destroy(NULL); +} + +#ifndef FLB_HAVE_YYJSON +static void test_tree_unavailable() +{ + /* without a tree backend callers keep their other way to read JSON */ + TEST_CHECK(flb_json_tree_read("{}", 2) == NULL); +} +#endif + +TEST_LIST = { +#ifdef FLB_HAVE_YYJSON + { "tree_types", test_tree_types }, + { "tree_nested_iteration", test_tree_nested_iteration }, + { "tree_invalid_unicode", test_tree_invalid_unicode }, + { "tree_rejects", test_tree_rejects }, +#else + { "tree_unavailable", test_tree_unavailable }, +#endif + { "tree_null_nodes", test_tree_null_nodes }, + { 0 } +}; From a52fcf3099b2d9f7b099842cafa1def9b8485cc7 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 31 Oct 2026 11:11:45 -0600 Subject: [PATCH 32/33] tests: internal: cover lenient OTLP JSON logs payloads An unpaired surrogate is decoded on the CFL path with the same records as the established conversion, content after the request object is left to the established conversion. Signed-off-by: Eduardo Silva --- tests/internal/opentelemetry.c | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index fa40f2262ce..a7c886fe1f0 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -1849,6 +1849,55 @@ static void json_cfl_check_parity(const char *name, const char *json, size_t lim flb_log_event_encoder_destroy(&cfl_enc); } +/* + * Invalid Unicode is replaced on both paths with the same records. Content + * after the request object is left to the established conversion, which + * accepts it with yyjson. Without yyjson every payload is left to it. + */ +void test_opentelemetry_json_cfl_lenient_payloads() +{ + int ret; + int error_status; + struct flb_log_event_encoder enc; + static const char *invalid_unicode = + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[{\"body\":" + "{\"stringValue\":\"bad \\uD800 surrogate\"}}]}]}]}"; + static const char *trailing = + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[{\"body\":" + "{\"stringValue\":\"text\"}}]}]}]} trailing"; + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + error_status = 0; + ret = json_cfl_encode(&enc, invalid_unicode, strlen(invalid_unicode), SIZE_MAX, + &error_status); +#ifdef FLB_HAVE_YYJSON + TEST_CHECK_(ret == 0 && enc.output_length > 0, + "invalid unicode: CFL result %i (%i)", ret, error_status); +#else + TEST_CHECK_(ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED, + "invalid unicode: CFL result %i", ret); +#endif + flb_log_event_encoder_destroy(&enc); + json_cfl_check_parity("invalid unicode", invalid_unicode, SIZE_MAX); + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + error_status = 0; + ret = json_cfl_encode(&enc, trailing, strlen(trailing), SIZE_MAX, &error_status); + TEST_CHECK_(ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED, + "trailing content: CFL result %i", ret); + flb_log_event_encoder_destroy(&enc); + +#ifdef FLB_HAVE_YYJSON + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + error_status = 0; + ret = flb_opentelemetry_logs_json_to_msgpack(&enc, trailing, strlen(trailing), + NULL, &error_status); + TEST_CHECK_(ret == 0 && enc.output_length > 0, + "trailing content: msgpack result %i (%i)", ret, error_status); + flb_log_event_encoder_destroy(&enc); +#endif +} + /* Payloads that exercise the leniency of the established conversion */ void test_opentelemetry_json_cfl_parity_values() { @@ -4528,6 +4577,8 @@ TEST_LIST = { { "opentelemetry_cases", test_opentelemetry_cases }, { "opentelemetry_cases_json_cfl_parity", test_opentelemetry_cases_json_cfl_parity }, { "opentelemetry_json_cfl_parity_values", test_opentelemetry_json_cfl_parity_values }, + { "opentelemetry_json_cfl_lenient_payloads", + test_opentelemetry_json_cfl_lenient_payloads }, { "opentelemetry_json_cfl_parity_corpus", test_opentelemetry_json_cfl_parity_corpus }, { "opentelemetry_protobuf_cfl_parity_corpus", test_opentelemetry_protobuf_cfl_parity_corpus }, From 5f1151bca36b83f9d142b509972ac705189542b2 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 31 Oct 2026 11:11:45 -0600 Subject: [PATCH 33/33] tests: integration: cover OTLP JSON logs with invalid Unicode A request with an unpaired surrogate is accepted and delivered without processors, with processors and with processors in HTTP workers. Signed-off-by: Eduardo Silva --- .../tests/test_in_opentelemetry_001.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py index 1a66dd5b89e..f6d7e688ff3 100644 --- a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py +++ b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py @@ -3408,3 +3408,31 @@ def test_in_opentelemetry_logs_direct_cfl_limit_parity(tmp_path, limit, encoding service.stop() assert statuses[1] == statuses[0] + + +@pytest.mark.parametrize( + "config_file", + [ + "otlp_http1_cleartext_workers.yaml", + "otlp_http1_cleartext.yaml", + IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_PROCESSORS_CONFIG, + ], + ids=["no_processors", "processors", "worker_processors"], +) +def test_in_opentelemetry_json_logs_replace_invalid_unicode(config_file): + service = Service(config_file) + payload = ( + b'{"resourceLogs":[{"scopeLogs":[{"logRecords":[{"body":' + b'{"stringValue":"bad \\uD800 surrogate"}}]}]}]}' + ) + + service.start() + try: + logs_before = len(data_storage["logs"]) + response = service.send_raw_request( + "/v1/logs", payload, content_type="application/json" + ) + assert response.status_code == 201 + service.wait_for_signal_count("logs", logs_before + 1, timeout=10) + finally: + service.stop()