From b2576b85b6a7e2643968542d69ad8f897b36995e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 01/61] lib: cmetrics: preserve typed OTLP metadata in msgpack Carry typed OTLP data point and family metadata (typed attributes, flags, exemplars, start times) as an optional sidecar of the internal msgpack format. Histogram buckets stay cumulative on the wire so 5.1 readers and writers keep interoperating, unknown layout markers are ignored, and samples with metadata merge by labels in cmt_cat. Signed-off-by: Eduardo Silva --- .../include/cmetrics/cmt_decode_msgpack.h | 1 + lib/cmetrics/include/cmetrics/cmt_map.h | 2 + lib/cmetrics/include/cmetrics/cmt_metric.h | 15 + .../include/cmetrics/cmt_otlp_metadata.h | 34 + .../include/cmetrics/cmt_variant_utils.h | 2 + lib/cmetrics/src/CMakeLists.txt | 1 + lib/cmetrics/src/cmt_cat.c | 169 ++++- lib/cmetrics/src/cmt_decode_msgpack.c | 47 +- lib/cmetrics/src/cmt_decode_opentelemetry.c | 189 ++--- lib/cmetrics/src/cmt_encode_msgpack.c | 4 +- lib/cmetrics/src/cmt_encode_opentelemetry.c | 147 +++- lib/cmetrics/src/cmt_encode_text.c | 4 + lib/cmetrics/src/cmt_map.c | 9 +- lib/cmetrics/src/cmt_otlp_metadata.c | 645 ++++++++++++++++++ lib/cmetrics/tests/cat.c | 196 ++++++ .../tests/data/histogram-cumulative-v5.1.0.c | 39 ++ .../data/histogram-cumulative-v5.1.0.msgpack | Bin 0 -> 343 bytes lib/cmetrics/tests/data/histogram-v5.1.0.c | 33 + .../tests/data/histogram-v5.1.0.msgpack | Bin 0 -> 250 bytes lib/cmetrics/tests/msgpack_abi.c | 221 ++++++ lib/cmetrics/tests/msgpack_temporality.c | 294 ++++++++ lib/cmetrics/tests/opentelemetry.c | 215 +++++- 22 files changed, 2172 insertions(+), 95 deletions(-) create mode 100644 lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h create mode 100644 lib/cmetrics/src/cmt_otlp_metadata.c create mode 100644 lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c create mode 100644 lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.msgpack create mode 100644 lib/cmetrics/tests/data/histogram-v5.1.0.c create mode 100644 lib/cmetrics/tests/data/histogram-v5.1.0.msgpack diff --git a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h index fd632247c6d..aed6097b2ad 100644 --- a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h +++ b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h @@ -55,6 +55,7 @@ struct cmt_msgpack_decode_context { uint64_t *summary_quantiles; size_t summary_quantiles_count; int aggregation_type; + int allow_reset; int metric_value_type_set; int metric_section_set; }; diff --git a/lib/cmetrics/include/cmetrics/cmt_map.h b/lib/cmetrics/include/cmetrics/cmt_map.h index 6a3c7e79620..2d64e9b729b 100644 --- a/lib/cmetrics/include/cmetrics/cmt_map.h +++ b/lib/cmetrics/include/cmetrics/cmt_map.h @@ -51,6 +51,8 @@ struct cmt_map { size_t indexed_metric_count; /* Most recently created metric; only changed with the metric list. */ struct cmt_metric *last_metric; + /* Owned family metadata; independent of metric names and sibling scopes. */ + struct cfl_kvlist *otlp_metadata; }; struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, diff --git a/lib/cmetrics/include/cmetrics/cmt_metric.h b/lib/cmetrics/include/cmetrics/cmt_metric.h index a45eaba5b50..8499851b4eb 100644 --- a/lib/cmetrics/include/cmetrics/cmt_metric.h +++ b/lib/cmetrics/include/cmetrics/cmt_metric.h @@ -75,6 +75,9 @@ struct cmt_metric { int hash_indexed; struct cmt_map *map; struct cfl_list _hash_head; + + /* Owned OTLP datapoint metadata; independent of string label identity. */ + struct cfl_kvlist *otlp_metadata; }; struct cmt_exp_histogram_snapshot { @@ -136,6 +139,18 @@ void cmt_metric_hist_set(struct cmt_metric *metric, uint64_t timestamp, uint64_t cmt_metric_hist_get_value(struct cmt_metric *metric, int bucket_id); +/* Convert cumulative native buckets without unsigned underflow. */ +static inline uint64_t cmt_metric_hist_get_bucket_count(struct cmt_metric *metric, int bucket_id) +{ + uint64_t current; + uint64_t previous; + + current = cmt_metric_hist_get_value(metric, bucket_id); + previous = bucket_id == 0 ? 0 : cmt_metric_hist_get_value(metric, bucket_id - 1); + return current >= previous ? current - previous : 0; +} + + double cmt_metric_hist_get_sum_value(struct cmt_metric *metric); uint64_t cmt_metric_hist_get_count_value(struct cmt_metric *metric); diff --git a/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h b/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h new file mode 100644 index 00000000000..0c6aa0472e3 --- /dev/null +++ b/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h @@ -0,0 +1,34 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CMT_OTLP_METADATA_H +#define CMT_OTLP_METADATA_H + +#include +#include +#include + +struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source); +int cmt_otlp_metadata_pack(mpack_writer_t *writer, struct cmt *context); +int cmt_otlp_metadata_restore(struct cmt *context); +int cmt_otlp_histogram_restore(struct cmt *context); +int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample); +int cmt_otlp_attributes_match_labels(struct cmt_map *map, struct cmt_metric *sample); + +#endif diff --git a/lib/cmetrics/include/cmetrics/cmt_variant_utils.h b/lib/cmetrics/include/cmetrics/cmt_variant_utils.h index 922f870fcea..8b250419a7d 100644 --- a/lib/cmetrics/include/cmetrics/cmt_variant_utils.h +++ b/lib/cmetrics/include/cmetrics/cmt_variant_utils.h @@ -477,6 +477,7 @@ static inline int unpack_cfl_variant_string(mpack_reader_t *reader, } (*value)->type = CFL_VARIANT_STRING; + (*value)->size = value_length; return 0; } @@ -528,6 +529,7 @@ static inline int unpack_cfl_variant_binary(mpack_reader_t *reader, } (*value)->type = CFL_VARIANT_BYTES; + (*value)->size = value_length; return 0; } diff --git a/lib/cmetrics/src/CMakeLists.txt b/lib/cmetrics/src/CMakeLists.txt index a4bed296713..f73ed74dcb2 100644 --- a/lib/cmetrics/src/CMakeLists.txt +++ b/lib/cmetrics/src/CMakeLists.txt @@ -15,6 +15,7 @@ set(src cmt_summary.c cmt_histogram.c cmt_exp_histogram.c + cmt_otlp_metadata.c cmt_metric.c cmt_metric_histogram.c cmt_map.c diff --git a/lib/cmetrics/src/cmt_cat.c b/lib/cmetrics/src/cmt_cat.c index 2ab52f01274..40e090127b8 100644 --- a/lib/cmetrics/src/cmt_cat.c +++ b/lib/cmetrics/src/cmt_cat.c @@ -17,6 +17,7 @@ * limitations under the License. */ +#include #include #include #include @@ -500,6 +501,151 @@ static inline void cat_scalar_value(struct cmt_metric *metric_dst, } } +static int copy_point_metadata(struct cmt_metric *dst, struct cmt_metric *src) +{ + struct cfl_kvlist *copy; + + if (src->otlp_metadata == NULL) { + return 0; + } + + copy = cmt_otlp_metadata_clone(src->otlp_metadata); + if (copy == NULL) { + return -1; + } + + cfl_kvlist_destroy(dst->otlp_metadata); + dst->otlp_metadata = copy; + + return 0; +} + +static int sample_labels_match(struct cmt_metric *metric, int count, char **labels) +{ + int index; + struct cfl_list *head; + struct cmt_map_label *label; + + index = 0; + cfl_list_foreach(head, &metric->labels) { + if (index >= count) { + return CMT_FALSE; + } + + label = cfl_list_entry(head, struct cmt_map_label, _head); + if ((label->name == NULL) != (labels[index] == NULL)) { + return CMT_FALSE; + } + if (label->name != NULL && strcmp(label->name, labels[index]) != 0) { + return CMT_FALSE; + } + index++; + } + + return index == count; +} + +/* OTLP permits several data points of one series in a single request when + * their time windows differ; those are distinct points, not duplicates. */ +static int same_data_point(struct cmt_metric *a, struct cmt_metric *b) +{ + if (cmt_metric_get_timestamp(a) != cmt_metric_get_timestamp(b)) { + return CMT_FALSE; + } + + if (cmt_metric_has_start_timestamp(a) != cmt_metric_has_start_timestamp(b)) { + return CMT_FALSE; + } + + if (cmt_metric_has_start_timestamp(a) && + cmt_metric_get_start_timestamp(a) != cmt_metric_get_start_timestamp(b)) { + return CMT_FALSE; + } + + return CMT_TRUE; +} + +static struct cmt_metric *append_point_sample(struct cmt_map *dst, + struct cmt_metric *src, + int count, char **labels) +{ + int index; + struct cmt_metric *metric; + struct cmt_map_label *label; + + metric = calloc(1, sizeof(struct cmt_metric)); + if (metric == NULL) { + cmt_errno(); + return NULL; + } + + cfl_list_init(&metric->labels); + cfl_list_init(&metric->_hash_head); + metric->map = dst; + metric->hash = src->hash; + cfl_list_add(&metric->_head, &dst->metrics); + + for (index = 0; index < count; index++) { + label = calloc(1, sizeof(struct cmt_map_label)); + if (label == NULL) { + cmt_errno(); + cmt_map_metric_destroy(metric); + return NULL; + } + + if (labels[index] != NULL) { + label->name = cfl_sds_create(labels[index]); + if (label->name == NULL) { + free(label); + cmt_map_metric_destroy(metric); + return NULL; + } + } + cfl_list_add(&label->_head, &metric->labels); + } + + return metric; +} + +/* + * Resolve the destination for a sample carrying OTLP metadata. Like any other + * sample it merges into an existing sample with the same labels, so contexts + * concatenated more than once never duplicate a series. A new sample is only + * appended for a distinct data point (different time window) of a series. + */ +static struct cmt_metric *get_point_sample(struct cmt_opts *opts, struct cmt_map *dst, + struct cmt_metric *src, + int count, char **labels) +{ + struct cfl_list *head; + struct cmt_metric *metric; + + if (count == 0) { + /* Label-less points always merge into the static metric. */ + return cmt_map_metric_get(opts, dst, 0, NULL, CMT_TRUE); + } + + metric = cmt_map_metric_get(opts, dst, count, labels, CMT_FALSE); + if (metric == NULL) { + /* No sample has these labels yet: create an indexed one. */ + return cmt_map_metric_get(opts, dst, count, labels, CMT_TRUE); + } + + if (same_data_point(metric, src)) { + return metric; + } + + /* Rare path: a series with several data points. */ + cfl_list_foreach(head, &dst->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (sample_labels_match(metric, count, labels) && same_data_point(metric, src)) { + return metric; + } + } + + return append_point_sample(dst, src, count, labels); +} + int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map *src) { int c; @@ -508,10 +654,20 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map struct cfl_list *head; struct cmt_metric *metric_dst; struct cmt_metric *metric_src; + struct cfl_kvlist *metadata; struct cmt_summary *summary; struct cmt_histogram *histogram_src; struct cmt_histogram *histogram_dst; + if (src->otlp_metadata) { + metadata = cmt_otlp_metadata_clone(src->otlp_metadata); + if (!metadata) { + return -1; + } + cfl_kvlist_destroy(dst->otlp_metadata); + dst->otlp_metadata = metadata; + } + /* Handle static metric (no labels case) */ if (src->metric_static_set) { dst->metric_static_set = CMT_TRUE; @@ -542,6 +698,9 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } } + if (copy_point_metadata(metric_dst, metric_src) != 0) { + return -1; + } cat_scalar_value(metric_dst, metric_src); } @@ -555,7 +714,12 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } c = cfl_list_size(&metric_src->labels); - metric_dst = cmt_map_metric_get(opts, dst, c, labels, CMT_TRUE); + if (metric_src->otlp_metadata != NULL) { + metric_dst = get_point_sample(opts, dst, metric_src, c, labels); + } + else { + metric_dst = cmt_map_metric_get(opts, dst, c, labels, CMT_TRUE); + } free(labels); if (!metric_dst) { @@ -584,6 +748,9 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } } + if (copy_point_metadata(metric_dst, metric_src) != 0) { + return -1; + } cat_scalar_value(metric_dst, metric_src); } diff --git a/lib/cmetrics/src/cmt_decode_msgpack.c b/lib/cmetrics/src/cmt_decode_msgpack.c index 59f12da6218..007e847a6ad 100644 --- a/lib/cmetrics/src/cmt_decode_msgpack.c +++ b/lib/cmetrics/src/cmt_decode_msgpack.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -1220,6 +1221,16 @@ static int unpack_metric_hash(mpack_reader_t *reader, size_t index, void *contex return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->hash); } +static int unpack_metric_otlp(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context = context; + + if (decode_context->metric->otlp_metadata != NULL) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + return unpack_cfl_kvlist(reader, &decode_context->metric->otlp_metadata); +} + static int unpack_metric(mpack_reader_t *reader, struct cmt_msgpack_decode_context *decode_context, struct cmt_metric **out_metric) @@ -1241,6 +1252,7 @@ static int unpack_metric(mpack_reader_t *reader, {"histogram", unpack_metric_histogram}, {"exp_histogram", unpack_metric_exp_histogram}, {"hash", unpack_metric_hash}, + {"otlp", unpack_metric_otlp}, {NULL, NULL} }; @@ -1328,6 +1340,8 @@ static int unpack_metric(mpack_reader_t *reader, free(metric->exp_hist_negative_buckets); } + cfl_kvlist_destroy(metric->otlp_metadata); + free(metric); } else { @@ -1362,8 +1376,8 @@ static int unpack_metric_array_entry(mpack_reader_t *reader, size_t index, void old_negative_buckets = NULL; if (0 == cfl_list_size(&metric->labels)) { - /* Should we care about finding more than one "implicitly static metric" in - * the array? + /* Every label-less entry is the static metric; when several are + * present the last one wins, exactly as released readers do. */ decode_context->map->metric_static_set = 1; @@ -1440,6 +1454,9 @@ static int unpack_metric_array_entry(mpack_reader_t *reader, size_t index, void cmt_atomic_load(&metric->val_int64)); cmt_atomic_store(&decode_context->map->metric.val_uint64, cmt_atomic_load(&metric->val_uint64)); + cfl_kvlist_destroy(decode_context->map->metric.otlp_metadata); + decode_context->map->metric.otlp_metadata = metric->otlp_metadata; + metric->otlp_metadata = NULL; decode_context->map->metric.hash = metric->hash; cmt_atomic_store(&decode_context->map->metric.timestamp, cmt_atomic_load(&metric->timestamp)); @@ -1545,6 +1562,24 @@ static int unpack_meta_aggregation_type(mpack_reader_t *reader, size_t index, vo return result; } +static int unpack_meta_allow_reset(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + uint64_t value; + int result; + + decode_context = context; + result = cmt_mpack_consume_uint_tag(reader, &value); + if (result != CMT_DECODE_MSGPACK_SUCCESS) { + return result; + } + if (value > 1) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + decode_context->allow_reset = value; + return CMT_DECODE_MSGPACK_SUCCESS; +} + static int unpack_meta_opts(mpack_reader_t *reader, size_t index, void *context) { struct cmt_msgpack_decode_context *decode_context; @@ -1705,6 +1740,7 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co {"buckets", unpack_meta_buckets}, {"quantiles", unpack_meta_quantiles}, {"aggregation_type", unpack_meta_aggregation_type}, + {"allow_reset", unpack_meta_allow_reset}, {NULL, NULL} }; @@ -1761,6 +1797,7 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co else if(decode_context->map->type == CMT_COUNTER) { counter = (struct cmt_counter *) decode_context->map->parent; counter->aggregation_type = decode_context->aggregation_type; + counter->allow_reset = decode_context->allow_reset; } } @@ -2245,6 +2282,12 @@ int cmt_decode_msgpack_create(struct cmt **out_cmt, char *in_buf, size_t in_size mpack_reader_init_data(&reader, &in_buf[*offset], in_size); result = unpack_context(&reader, cmt); + if (result == CMT_DECODE_MSGPACK_SUCCESS && cmt_otlp_histogram_restore(cmt) != 0) { + result = CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; + } + if (result == CMT_DECODE_MSGPACK_SUCCESS && cmt_otlp_metadata_restore(cmt) != 0) { + result = CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; + } remainder = mpack_reader_remaining(&reader, NULL); diff --git a/lib/cmetrics/src/cmt_decode_opentelemetry.c b/lib/cmetrics/src/cmt_decode_opentelemetry.c index 6aafa4abbbf..2bb3e460257 100644 --- a/lib/cmetrics/src/cmt_decode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_decode_opentelemetry.c @@ -20,6 +20,7 @@ #include "cmt_protobuf.h" #include #include +#include #include #include #include @@ -35,6 +36,7 @@ #include #include #include +#include static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source); @@ -341,81 +343,63 @@ static uint64_t compute_metric_hash(struct cmt_map *map, struct cmt_metric *samp return cfl_hash_64bits_digest(&state); } -static char *map_type_to_key(int map_type) +static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct cmt_map *map) { - switch (map_type) { - case CMT_COUNTER: - return "counter"; - case CMT_GAUGE: - return "gauge"; - case CMT_UNTYPED: - return "untyped"; - case CMT_SUMMARY: - return "summary"; - case CMT_HISTOGRAM: - return "histogram"; - case CMT_EXP_HISTOGRAM: - return "exp_histogram"; - default: - return "unknown"; + if (!map->otlp_metadata) { + map->otlp_metadata = cfl_kvlist_create(); } + return map->otlp_metadata; } -static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct cmt_map *map) +static struct cfl_kvlist *get_or_create_data_point_metadata_context(struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample, + uint64_t timestamp) { - struct cfl_kvlist *otlp_root; - struct cfl_kvlist *metrics_root; - struct cfl_kvlist *type_root; - - if (cmt == NULL || map == NULL || map->opts == NULL || map->opts->fqname == NULL) { + if (sample == NULL) { return NULL; } - - otlp_root = get_or_create_external_metadata_kvlist(cmt->external_metadata, "otlp"); - if (otlp_root == NULL) { - return NULL; - } - - metrics_root = get_or_create_external_metadata_kvlist(otlp_root, "metrics"); - if (metrics_root == NULL) { - return NULL; + if (sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { + sample->hash = compute_metric_hash(map, sample); } - - type_root = get_or_create_external_metadata_kvlist(metrics_root, map_type_to_key(map->type)); - if (type_root == NULL) { - return NULL; + if (sample->otlp_metadata == NULL) { + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata && cmt_otlp_capture_labels(map, sample) != 0) { + return NULL; + } } - - return get_or_create_external_metadata_kvlist(type_root, map->opts->fqname); + return sample->otlp_metadata; } -static struct cfl_kvlist *get_or_create_data_point_metadata_context(struct cmt *cmt, - struct cmt_map *map, - struct cmt_metric *sample, - uint64_t timestamp) +/* Keep telemetry attributes typed; string labels remain available to native exporters. */ +static int clone_point_attributes(struct cfl_kvlist *metadata, + Opentelemetry__Proto__Common__V1__KeyValue **attributes, + size_t count) { - char key[128]; - struct cfl_kvlist *metric_context; - struct cfl_kvlist *datapoints_context; + struct cfl_kvlist *copy; + size_t index; - if (sample != NULL && sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { - sample->hash = compute_metric_hash(map, sample); + copy = cfl_kvlist_create(); + if (copy == NULL) { + return -1; } - - metric_context = get_or_create_metric_metadata_context(cmt, map); - if (metric_context == NULL) { - return NULL; + for (index = 0; index < count; index++) { + if (attributes[index] != NULL && + (attributes[index]->value == NULL || + attributes[index]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET)) { + continue; + } + if (attributes[index] == NULL || clone_kvlist_entry(copy, attributes[index]) != 0) { + cfl_kvlist_destroy(copy); + return -1; + } } - - datapoints_context = get_or_create_external_metadata_kvlist(metric_context, "datapoints"); - if (datapoints_context == NULL) { - return NULL; + if (cfl_kvlist_insert_kvlist(metadata, "attributes", copy) != 0) { + cfl_kvlist_destroy(copy); + return -1; } - - snprintf(key, sizeof(key) - 1, "%" PRIx64 ":%" PRIu64, - sample != NULL ? sample->hash : 0, timestamp); - - return get_or_create_external_metadata_kvlist(datapoints_context, key); + return 0; } static int clone_exemplars_to_kvlist(struct cfl_kvlist *target, @@ -724,6 +708,30 @@ static int decode_data_point_labels(struct cmt *cmt, return result; } +/* A repeated label-less point replaces the static sample's previous point. */ +static void reset_static_data_point(struct cmt_map *map) +{ + struct cmt_metric *metric; + struct cmt_histogram *histogram; + + metric = &map->metric; + + if (metric->otlp_metadata != NULL) { + cfl_kvlist_destroy(metric->otlp_metadata); + metric->otlp_metadata = NULL; + } + + if (map->type == CMT_HISTOGRAM && metric->hist_buckets != NULL) { + histogram = (struct cmt_histogram *) map->parent; + memset(metric->hist_buckets, 0, + sizeof(uint64_t) * (histogram->buckets->count + 1)); + } + else if (map->type == CMT_SUMMARY && metric->sum_quantiles != NULL) { + memset(metric->sum_quantiles, 0, + sizeof(uint64_t) * metric->sum_quantiles_count); + } +} + static int decode_numerical_data_point(struct cmt *cmt, struct cmt_map *map, Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point) @@ -737,9 +745,9 @@ static int decode_numerical_data_point(struct cmt *cmt, result = CMT_DECODE_OPENTELEMETRY_SUCCESS; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -768,7 +776,9 @@ static int decode_numerical_data_point(struct cmt *cmt, } else { sample = &map->metric; - + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -799,6 +809,10 @@ static int decode_numerical_data_point(struct cmt *cmt, } point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); @@ -909,9 +923,9 @@ static int decode_summary_data_point(struct cmt *cmt, static_metric_detected = CMT_FALSE; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -942,7 +956,9 @@ static int decode_summary_data_point(struct cmt *cmt, } else { sample = &map->metric; - + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -973,6 +989,10 @@ static int decode_summary_data_point(struct cmt *cmt, data_point->count); point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); @@ -1022,6 +1042,7 @@ static int decode_histogram_data_point(struct cmt *cmt, { int static_metric_detected; struct cmt_histogram *histogram; + uint64_t cumulative = 0; struct cmt_metric *sample; int result; size_t index; @@ -1079,9 +1100,9 @@ static int decode_histogram_data_point(struct cmt *cmt, static_metric_detected = CMT_FALSE; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -1112,7 +1133,9 @@ static int decode_histogram_data_point(struct cmt *cmt, } else { sample = &map->metric; - + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -1135,8 +1158,11 @@ static int decode_histogram_data_point(struct cmt *cmt, for (index = 0 ; index < data_point->n_bucket_counts; index++) { - cmt_metric_hist_set(sample, data_point->time_unix_nano, - index, data_point->bucket_counts[index]); + if (UINT64_MAX - cumulative < data_point->bucket_counts[index]) { + return CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; + } + cumulative += data_point->bucket_counts[index]; + cmt_metric_hist_set(sample, data_point->time_unix_nano, index, cumulative); } cmt_metric_hist_sum_set(sample, data_point->time_unix_nano, @@ -1145,6 +1171,10 @@ static int decode_histogram_data_point(struct cmt *cmt, data_point->count); point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); @@ -1345,9 +1375,9 @@ static int decode_exponential_histogram_data_point(struct cmt *cmt, static_metric_detected = CMT_FALSE; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -1376,6 +1406,9 @@ static int decode_exponential_histogram_data_point(struct cmt *cmt, } else { sample = &map->metric; + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -1438,6 +1471,10 @@ static int decode_exponential_histogram_data_point(struct cmt *cmt, struct cfl_kvlist *point_metadata; point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); diff --git a/lib/cmetrics/src/cmt_encode_msgpack.c b/lib/cmetrics/src/cmt_encode_msgpack.c index 4341b9c651f..bdf9ffe7ebf 100644 --- a/lib/cmetrics/src/cmt_encode_msgpack.c +++ b/lib/cmetrics/src/cmt_encode_msgpack.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -264,6 +265,7 @@ static int pack_metric(mpack_writer_t *writer, struct cmt_map *map, struct cmt_m for (index = 0 ; index <= histogram->buckets->count ; index++) { + /* Wire buckets stay cumulative, as every released reader expects. */ mpack_write_uint(writer, cmt_metric_hist_get_value(metric, index)); } @@ -477,7 +479,7 @@ static int pack_context_header(mpack_writer_t *writer, struct cmt *cmt) mpack_start_map(writer, 3); mpack_write_cstr(writer, "cmetrics"); - result = pack_cfl_variant_kvlist(writer, cmt->internal_metadata); + result = cmt_otlp_metadata_pack(writer, cmt); if (result != 0) { return -1; diff --git a/lib/cmetrics/src/cmt_encode_opentelemetry.c b/lib/cmetrics/src/cmt_encode_opentelemetry.c index 65b6399ee40..91dd4ac7999 100644 --- a/lib/cmetrics/src/cmt_encode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_encode_opentelemetry.c @@ -18,6 +18,7 @@ */ #include #include +#include #include #include #include @@ -82,7 +83,7 @@ struct cfl_kvlist *fetch_metadata_kvlist_key(struct cfl_kvlist *kvlist, char *ke entry_variant = cfl_kvlist_fetch(kvlist, key); - if (entry_variant != NULL) { + if (entry_variant != NULL && entry_variant->type == CFL_VARIANT_KVLIST) { entry_kvlist = entry_variant->data.as_kvlist; } else { @@ -154,6 +155,10 @@ static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, stru return NULL; } + if (map->otlp_metadata) { + return map->otlp_metadata; + } + otlp_root = fetch_metadata_kvlist_key(cmt->external_metadata, "otlp"); if (otlp_root == NULL) { return NULL; @@ -180,6 +185,10 @@ static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, struct cfl_kvlist *metric_context; struct cfl_kvlist *datapoints_context; + if (sample != NULL && sample->otlp_metadata != NULL) { + return sample->otlp_metadata; + } + metric_context = get_metric_otlp_metadata_context(cmt, map); if (metric_context == NULL) { return NULL; @@ -541,7 +550,10 @@ static size_t resolve_target_scope_index(struct cmt_opentelemetry_context *conte return 0; } - metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + metadata = fetch_metadata_kvlist_key(metric_context, "routing"); + if (metadata == NULL) { + metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + } if (metadata == NULL) { return 0; } @@ -1294,6 +1306,108 @@ static inline Opentelemetry__Proto__Common__V1__KeyValue **cfl_kvlist_to_otlp_kv } +static void *attribute_arena_alloc(void *data, size_t size) +{ + return cfl_arena_malloc(data, size); +} + +static void attribute_arena_free(void *data, void *pointer) +{ + /* The encoder releases all datapoint objects with their arena. */ +} + +static int apply_typed_point_attributes(struct cmt_opentelemetry_context *context, + struct cmt_map *map, struct cmt_metric *sample, + void *data_point) +{ + struct cfl_kvlist *metadata; + struct cfl_kvlist *attributes; + struct cfl_arena *arena; + struct cfl_list *head; + struct cmt_label *label; + ProtobufCAllocator allocator; + Opentelemetry__Proto__Common__V1__KeyValue **heap; + Opentelemetry__Proto__Common__V1__KeyValue **replacement; + Opentelemetry__Proto__Common__V1__KeyValue ***target; + size_t *target_count; + uint8_t *buffer; + size_t count; + size_t size; + size_t index; + size_t static_count; + + metadata = get_data_point_otlp_metadata_context(context->cmt, map, sample); + attributes = fetch_metadata_kvlist_key(metadata, "attributes"); + if (attributes == NULL || !cmt_otlp_attributes_match_labels(map, sample)) { + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; + } + count = cfl_kvlist_count(attributes); + static_count = cfl_list_size(&context->cmt->static_labels->list); + heap = cfl_kvlist_to_otlp_kvpair_list(attributes); + if (heap == NULL) { + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + arena = get_context_arena(context); + allocator.alloc = attribute_arena_alloc; + allocator.free = attribute_arena_free; + allocator.allocator_data = arena; + replacement = cfl_arena_calloc(arena, count + static_count + 1, sizeof(*replacement)); + if (replacement == NULL) { + otlp_kvpair_list_destroy(heap, count); + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + for (index = 0; index < count; index++) { + size = opentelemetry__proto__common__v1__key_value__get_packed_size(heap[index]); + buffer = malloc(size ? size : 1); + if (buffer == NULL) { + otlp_kvpair_list_destroy(heap, count); + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + opentelemetry__proto__common__v1__key_value__pack(heap[index], buffer); + replacement[index] = opentelemetry__proto__common__v1__key_value__unpack( + &allocator, size, buffer); + free(buffer); + if (replacement[index] == NULL) { + otlp_kvpair_list_destroy(heap, count); + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + } + otlp_kvpair_list_destroy(heap, count); + index = count; + cfl_list_foreach(head, &context->cmt->static_labels->list) { + label = cfl_list_entry(head, struct cmt_label, _head); + replacement[index] = initialize_string_attribute(arena, label->key, label->val); + if (replacement[index++] == NULL) { + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + } + switch (map->type) { + case CMT_COUNTER: + case CMT_GAUGE: + case CMT_UNTYPED: + target = &((Opentelemetry__Proto__Metrics__V1__NumberDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__NumberDataPoint *) data_point)->n_attributes; + break; + case CMT_HISTOGRAM: + target = &((Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *) data_point)->n_attributes; + break; + case CMT_EXP_HISTOGRAM: + target = &((Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *) data_point)->n_attributes; + break; + case CMT_SUMMARY: + target = &((Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *) data_point)->n_attributes; + break; + default: + return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; + } + *target = replacement; + *target_count = count + static_count; + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; +} + static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_kvlist_to_otlp_any_value(struct cfl_variant *value) { size_t entry_count; @@ -1453,6 +1567,7 @@ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_binary_to_ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_to_otlp_any_value(struct cfl_variant *value) { Opentelemetry__Proto__Common__V1__AnyValue *result; + struct cfl_variant signed_value; if (value->type == CFL_VARIANT_STRING) { result = cfl_variant_string_to_otlp_any_value(value); @@ -1463,6 +1578,17 @@ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_to_otlp_an else if (value->type == CFL_VARIANT_INT) { result = cfl_variant_int64_to_otlp_any_value(value); } + else if (value->type == CFL_VARIANT_UINT) { + if (value->data.as_uint64 > INT64_MAX) { + result = NULL; + } + else { + signed_value = *value; + signed_value.type = CFL_VARIANT_INT; + signed_value.data.as_int64 = (int64_t) value->data.as_uint64; + result = cfl_variant_int64_to_otlp_any_value(&signed_value); + } + } else if (value->type == CFL_VARIANT_DOUBLE) { result = cfl_variant_double_to_otlp_any_value(value); } @@ -1518,6 +1644,10 @@ static int64_t fetch_metadata_int64_key(struct cfl_kvlist *metadata, char *key_n return 0; } + if (value->type == CFL_VARIANT_UINT && value->data.as_uint64 <= INT64_MAX) { + return (int64_t) value->data.as_uint64; + } + if (value->type != CFL_VARIANT_INT) { *error_flag = CMT_TRUE; @@ -1751,11 +1881,11 @@ static Opentelemetry__Proto__Metrics__V1__ResourceMetrics ** void destroy_instrumentation_scope(Opentelemetry__Proto__Common__V1__InstrumentationScope *scope) { - if (scope->name != NULL) { + if (is_string_releaseable(scope->name)) { cfl_sds_destroy(scope->name); } - if (scope->version != NULL) { + if (is_string_releaseable(scope->version)) { cfl_sds_destroy(scope->version); } @@ -2367,7 +2497,9 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * if (bucket_list != NULL) { for (index = 0 ; index < bucket_count ; index++) { - data_point->bucket_counts[index] = bucket_list[index]; + data_point->bucket_counts[index] = index == 0 ? bucket_list[index] : + (bucket_list[index] >= bucket_list[index - 1] ? + bucket_list[index] - bucket_list[index - 1] : 0); } } } @@ -3417,6 +3549,11 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, } } + result = apply_typed_point_attributes(context, map, sample, data_point); + if (result != CMT_ENCODE_OPENTELEMETRY_SUCCESS) { + destroy_data_point(data_point, map->type); + return result; + } apply_data_point_metadata_from_otlp_context(context->cmt, map, sample, data_point); result = append_data_point_to_metric(metric, (void *) data_point, sample_index); diff --git a/lib/cmetrics/src/cmt_encode_text.c b/lib/cmetrics/src/cmt_encode_text.c index 7d499d72cee..1835c11461e 100644 --- a/lib/cmetrics/src/cmt_encode_text.c +++ b/lib/cmetrics/src/cmt_encode_text.c @@ -188,6 +188,10 @@ static struct cfl_kvlist *get_data_point_metadata_context(struct cmt *cmt, const char *type_key; char key[128]; + if (metric != NULL && metric->otlp_metadata != NULL) { + return metric->otlp_metadata; + } + type_key = map_type_to_otlp_key(map->type); if (type_key == NULL) { return NULL; diff --git a/lib/cmetrics/src/cmt_map.c b/lib/cmetrics/src/cmt_map.c index fb4fe9a150e..ccec20fc464 100644 --- a/lib/cmetrics/src/cmt_map.c +++ b/lib/cmetrics/src/cmt_map.c @@ -46,6 +46,8 @@ static void metric_release_storage(struct cmt_metric *metric) free(metric->exp_hist_positive_buckets); free(metric->exp_hist_negative_buckets); free(metric->sum_quantiles); + cfl_kvlist_destroy(metric->otlp_metadata); + metric->otlp_metadata = NULL; metric->hist_buckets = NULL; metric->exp_hist_positive_buckets = NULL; @@ -506,16 +508,13 @@ void cmt_map_destroy(struct cmt_map *map) } /* histogram and quantile allocation for static metric */ - if (map->metric_static_set) { - metric = &map->metric; - - metric_release_storage(metric); - } + metric_release_storage(&map->metric); if (map->unit != NULL) { cfl_sds_destroy(map->unit); } + cfl_kvlist_destroy(map->otlp_metadata); free(map->metric_buckets); free(map); diff --git a/lib/cmetrics/src/cmt_otlp_metadata.c b/lib/cmetrics/src/cmt_otlp_metadata.c new file mode 100644 index 00000000000..81ed8171b82 --- /dev/null +++ b/lib/cmetrics/src/cmt_otlp_metadata.c @@ -0,0 +1,645 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#define METADATA_KEY "__cmetrics_native_metadata_v1" +/* Written only by unreleased builds; ignored on input, never emitted. */ +#define HISTOGRAM_MARKER_KEY "__cmetrics_histogram_buckets" + +struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source) +{ + mpack_writer_t writer; + mpack_reader_t reader; + struct cfl_kvlist *copy = NULL; + char *buffer = NULL; + size_t size = 0; + int result; + + if (source == NULL) { + return NULL; + } + mpack_writer_init_growable(&writer, &buffer, &size); + result = pack_cfl_variant_kvlist(&writer, source); + if (mpack_writer_destroy(&writer) != mpack_ok || result != 0) { + free(buffer); + return NULL; + } + mpack_reader_init_data(&reader, buffer, size); + result = unpack_cfl_kvlist(&reader, ©); + if (mpack_reader_destroy(&reader) != mpack_ok || result != 0) { + cfl_kvlist_destroy(copy); + copy = NULL; + } + free(buffer); + return copy; +} + +static int visit_maps(struct cmt *context, + int (*callback)(struct cmt_map *, void *), void *data) +{ + struct cfl_list *head; + int result; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_summary *summary; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; + + + cfl_list_foreach(head, &context->counters) { + counter = cfl_list_entry(head, struct cmt_counter, _head); + result = callback(counter->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->gauges) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + result = callback(gauge->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->untypeds) { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + result = callback(untyped->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->summaries) { + summary = cfl_list_entry(head, struct cmt_summary, _head); + result = callback(summary->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + result = callback(histogram->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + result = callback(exp_histogram->map, data); + if (result != 0) { + return result; + } + } + return 0; +} + +static int count_maps(struct cmt_map *map, void *data) +{ + (*(size_t *) data)++; + return 0; +} + +static int has_metadata(struct cmt_map *map, void *data) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + if (map->otlp_metadata || map->metric.otlp_metadata || + (map->type == CMT_COUNTER && ((struct cmt_counter *) map->parent)->allow_reset)) { + return 1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (sample->otlp_metadata) { + return 1; + } + } + return 0; +} + +static int pack_optional_map(mpack_writer_t *writer, struct cfl_kvlist *map) +{ + mpack_writer_t metadata_writer; + char *buffer = NULL; + size_t size = 0; + int result; + + if (map == NULL) { + mpack_write_nil(writer); + return 0; + } + /* A binary sidecar does not add nesting to the established wire metadata + * tree. Older readers can retain/ignore it even for deeply nested values. */ + mpack_writer_init_growable(&metadata_writer, &buffer, &size); + result = pack_cfl_variant_kvlist(&metadata_writer, map); + if (mpack_writer_destroy(&metadata_writer) != mpack_ok || result || size > UINT32_MAX) { + free(buffer); + return -1; + } + mpack_write_bin(writer, buffer, size); + free(buffer); + return 0; +} + +static int pack_map_metadata(struct cmt_map *map, void *data) +{ + mpack_writer_t *writer = data; + struct cfl_list *head; + struct cmt_metric *sample; + + mpack_start_map(writer, 5); + mpack_write_cstr(writer, "type"); + mpack_write_int(writer, map->type); + mpack_write_cstr(writer, "name"); + mpack_write_cstr(writer, map->opts->fqname); + mpack_write_cstr(writer, "family"); + if (pack_optional_map(writer, map->otlp_metadata) != 0) { + return -1; + } + mpack_write_cstr(writer, "allow_reset"); + mpack_write_bool(writer, map->type == CMT_COUNTER && + ((struct cmt_counter *) map->parent)->allow_reset); + mpack_write_cstr(writer, "points"); + mpack_start_array(writer, cfl_list_size(&map->metrics) + !!map->metric_static_set); + if (map->metric_static_set && pack_optional_map(writer, map->metric.otlp_metadata) != 0) { + return -1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (pack_optional_map(writer, sample->otlp_metadata) != 0) { + return -1; + } + } + mpack_finish_array(writer); + mpack_finish_map(writer); + return 0; +} + +/* The established internal metadata map accepts arbitrary typed keys. No new + * metric/map wire fields are emitted, so old decoders still consume the chunk. + * Histogram buckets are never re-shaped here: they stay cumulative on the wire, + * exactly as released readers and writers expect. */ +int cmt_otlp_metadata_pack(mpack_writer_t *writer, struct cmt *context) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + size_t count = 0; + size_t entries; + int has_sidecar; + int has_marker; + + has_sidecar = visit_maps(context, has_metadata, NULL) != 0; + has_marker = cfl_kvlist_fetch(context->internal_metadata, + HISTOGRAM_MARKER_KEY) != NULL; + if (!has_sidecar && !has_marker) { + return pack_cfl_variant_kvlist(writer, context->internal_metadata); + } + if (has_sidecar && cfl_kvlist_fetch(context->internal_metadata, METADATA_KEY)) { + return -1; + } + + /* A stale bucket marker is never forwarded. */ + entries = has_sidecar; + cfl_list_foreach(head, &context->internal_metadata->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (strcmp(pair->key, HISTOGRAM_MARKER_KEY) != 0) { + entries++; + } + } + + mpack_start_map(writer, entries); + cfl_list_foreach(head, &context->internal_metadata->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (strcmp(pair->key, HISTOGRAM_MARKER_KEY) == 0) { + continue; + } + mpack_write_str(writer, pair->key, cfl_sds_len(pair->key)); + if (pack_cfl_variant(writer, pair->val) != 0) { + return -1; + } + } + + if (!has_sidecar) { + mpack_finish_map(writer); + return 0; + } + + mpack_write_cstr(writer, METADATA_KEY); + visit_maps(context, count_maps, &count); + mpack_start_array(writer, count); + if (visit_maps(context, pack_map_metadata, writer) != 0) { + return -1; + } + mpack_finish_array(writer); + mpack_finish_map(writer); + return 0; +} + +struct staged_family_metadata { + struct cmt_map *map; + struct cfl_kvlist *family; + struct cfl_kvlist **points; + size_t point_count; + int allow_reset; + int restore_points; +}; + +struct restore_state { + struct cfl_array *families; + size_t index; + struct staged_family_metadata *staged; +}; + +static int restore_optional_map(struct cfl_kvlist **target, struct cfl_variant *value) +{ + struct cfl_kvlist *copy = NULL; + mpack_reader_t reader; + size_t remaining; + int result; + + if (value == NULL || value->type == CFL_VARIANT_NULL) { + return 0; + } + if (value->type == CFL_VARIANT_BYTES) { + mpack_reader_init_data(&reader, value->data.as_bytes, cfl_sds_len(value->data.as_bytes)); + result = unpack_cfl_kvlist(&reader, ©); + remaining = result == 0 ? mpack_reader_remaining(&reader, NULL) : 0; + if (mpack_reader_destroy(&reader) != mpack_ok || result || remaining) { + cfl_kvlist_destroy(copy); + return -1; + } + } + else { + return -1; + } + if (!copy) { + return -1; + } + cfl_kvlist_destroy(*target); + *target = copy; + return 0; +} + +static int stage_map_metadata(struct cmt_map *map, void *data) +{ + struct restore_state *state = data; + struct staged_family_metadata *staged; + struct cfl_variant *family; + struct cfl_variant *points; + struct cfl_variant *reset; + size_t index; + + staged = &state->staged[state->index]; + family = state->families->entries[state->index++]; + staged->map = map; + points = cfl_kvlist_fetch(family->data.as_kvlist, "points"); + reset = cfl_kvlist_fetch(family->data.as_kvlist, "allow_reset"); + if (!reset || reset->type != CFL_VARIANT_BOOL || + restore_optional_map(&staged->family, cfl_kvlist_fetch(family->data.as_kvlist, "family"))) { + return -1; + } + staged->allow_reset = reset->data.as_bool; + + /* Label-less entries fold into the static metric on decode, so the point + * list of that family may no longer line up: keep the family metadata + * but skip its point metadata rather than misattributing it. */ + staged->restore_points = points->data.as_array->entry_count == + cfl_list_size(&map->metrics) + !!map->metric_static_set; + if (!staged->restore_points) { + return 0; + } + + staged->point_count = points->data.as_array->entry_count; + if (staged->point_count > 0) { + staged->points = calloc(staged->point_count, sizeof(*staged->points)); + if (!staged->points) { + return -1; + } + } + for (index = 0; index < staged->point_count; index++) { + if (restore_optional_map(&staged->points[index], points->data.as_array->entries[index])) { + return -1; + } + } + return 0; +} + +static void commit_map_metadata(struct staged_family_metadata *staged) +{ + struct cmt_map *map = staged->map; + struct cfl_list *head; + struct cmt_metric *sample; + size_t index = 0; + + cfl_kvlist_destroy(map->otlp_metadata); + map->otlp_metadata = staged->family; + staged->family = NULL; + if (map->type == CMT_COUNTER) { + ((struct cmt_counter *) map->parent)->allow_reset = staged->allow_reset; + } + if (!staged->restore_points) { + return; + } + if (map->metric_static_set) { + cfl_kvlist_destroy(map->metric.otlp_metadata); + map->metric.otlp_metadata = staged->points[index]; + staged->points[index++] = NULL; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = staged->points[index]; + staged->points[index++] = NULL; + } +} + +static int check_map_metadata_shape(struct cmt_map *map, void *data) +{ + struct restore_state *state = data; + struct cfl_variant *family; + struct cfl_variant *points; + struct cfl_variant *name; + struct cfl_variant *type; + + if (state->index >= state->families->entry_count) { + return -1; + } + family = state->families->entries[state->index++]; + if (family->type != CFL_VARIANT_KVLIST) { + return -1; + } + points = cfl_kvlist_fetch(family->data.as_kvlist, "points"); + name = cfl_kvlist_fetch(family->data.as_kvlist, "name"); + type = cfl_kvlist_fetch(family->data.as_kvlist, "type"); + if (points == NULL || points->type != CFL_VARIANT_ARRAY) { + return -1; + } + if (name != NULL && (name->type != CFL_VARIANT_STRING || + cfl_sds_len(name->data.as_string) != cfl_sds_len(map->opts->fqname) || + memcmp(name->data.as_string, map->opts->fqname, + cfl_sds_len(map->opts->fqname)) != 0)) { + return -1; + } + if (type != NULL && !((type->type == CFL_VARIANT_INT && type->data.as_int64 == map->type) || + (type->type == CFL_VARIANT_UINT && type->data.as_uint64 == map->type))) { + return -1; + } + return 0; +} + +int cmt_otlp_metadata_restore(struct cmt *context) +{ + struct cfl_variant *value; + struct restore_state state; + int result; + size_t family_index; + size_t point_index; + + value = cfl_kvlist_fetch(context->internal_metadata, METADATA_KEY); + if (value == NULL) { + return 0; + } + if (value->type != CFL_VARIANT_ARRAY) { + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + return 0; + } + state.families = value->data.as_array; + state.index = 0; + result = visit_maps(context, check_map_metadata_shape, &state); + if (result != 0 || state.index != state.families->entry_count) { + /* Optional metadata must not invalidate an otherwise usable chunk. */ + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + return 0; + } + state.staged = calloc(state.families->entry_count, sizeof(*state.staged)); + if (!state.staged && state.families->entry_count > 0) { + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + return 0; + } + state.index = 0; + result = visit_maps(context, stage_map_metadata, &state); + if (result == 0) { + for (family_index = 0; family_index < state.families->entry_count; family_index++) { + commit_map_metadata(&state.staged[family_index]); + } + } + for (family_index = 0; family_index < state.families->entry_count; family_index++) { + cfl_kvlist_destroy(state.staged[family_index].family); + if (state.staged[family_index].points != NULL) { + for (point_index = 0; point_index < state.staged[family_index].point_count; point_index++) { + cfl_kvlist_destroy(state.staged[family_index].points[point_index]); + } + } + free(state.staged[family_index].points); + } + free(state.staged); + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + /* Malformed optional metadata never invalidates the native metric payload. */ + return 0; +} + +int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample) +{ + struct cfl_list *key_head; + struct cfl_list *value_head; + struct cmt_map_label *key; + struct cmt_map_label *value; + struct cfl_kvlist *projection; + + if (!sample->otlp_metadata) { + return -1; + } + projection = cfl_kvlist_create(); + if (!projection) { + return -1; + } + key_head = map->label_keys.next; + value_head = sample->labels.next; + while (key_head != &map->label_keys && value_head != &sample->labels) { + key = cfl_list_entry(key_head, struct cmt_map_label, _head); + value = cfl_list_entry(value_head, struct cmt_map_label, _head); + if (key->name && value->name && cfl_kvlist_insert_string(projection, key->name, value->name)) { + cfl_kvlist_destroy(projection); + return -1; + } + key_head = key_head->next; + value_head = value_head->next; + } + cfl_kvlist_remove(sample->otlp_metadata, "label_projection"); + if (cfl_kvlist_insert_kvlist(sample->otlp_metadata, "label_projection", projection)) { + cfl_kvlist_destroy(projection); + return -1; + } + return 0; +} + +int cmt_otlp_attributes_match_labels(struct cmt_map *map, struct cmt_metric *sample) +{ + struct cfl_variant *projection; + struct cfl_list *key_head; + struct cfl_list *value_head; + struct cfl_list *original_head; + struct cfl_kvpair *original; + struct cmt_map_label *key; + struct cmt_map_label *value; + + if (!sample->otlp_metadata) { + return CMT_FALSE; + } + projection = cfl_kvlist_fetch(sample->otlp_metadata, "label_projection"); + if (!projection || projection->type != CFL_VARIANT_KVLIST) { + return CMT_FALSE; + } + key_head = map->label_keys.next; + value_head = sample->labels.next; + original_head = projection->data.as_kvlist->list.next; + while (key_head != &map->label_keys && value_head != &sample->labels) { + key = cfl_list_entry(key_head, struct cmt_map_label, _head); + value = cfl_list_entry(value_head, struct cmt_map_label, _head); + if (key->name && value->name) { + if (original_head == &projection->data.as_kvlist->list) { + return CMT_FALSE; + } + original = cfl_list_entry(original_head, struct cfl_kvpair, _head); + if (original->val->type != CFL_VARIANT_STRING || + strcmp(original->key, key->name) || strcmp(original->val->data.as_string, value->name)) { + return CMT_FALSE; + } + original_head = original_head->next; + } + key_head = key_head->next; + value_head = value_head->next; + } + return original_head == &projection->data.as_kvlist->list && value_head == &sample->labels; +} + +/* + * CMetrics stores classic histogram buckets cumulatively (including +Inf) and + * the msgpack wire format carries them exactly like that. Fluent Bit v5.1.0 + * and older, however, stored OTLP-decoded histograms with independent bucket + * counts in the same fields, so chunks written by those releases (forward + * peers, filesystem backlog) can contain either shape without any marker. + * + * Detect the shape from the sample itself instead of trusting a marker: + * + * - monotonic buckets whose last (+Inf) value equals count: cumulative. + * - otherwise, buckets whose sum equals count: independent counts. + * - anything else is left untouched (cumulative by default). + * + * Both rules can only match at once when every bucket but +Inf is zero, in + * which case both interpretations are identical, so the check is unambiguous + * for consistent data. + */ +static int histogram_sample_is_independent(struct cmt_metric *sample, size_t count) +{ + uint64_t total; + uint64_t value; + uint64_t previous; + uint64_t sample_count; + size_t index; + int monotonic; + + if (sample->hist_buckets == NULL || count == 0) { + return CMT_FALSE; + } + + sample_count = cmt_metric_hist_get_count_value(sample); + total = 0; + previous = 0; + monotonic = CMT_TRUE; + + for (index = 0; index < count; index++) { + value = sample->hist_buckets[index]; + if (value < previous) { + monotonic = CMT_FALSE; + } + previous = value; + + if (value > UINT64_MAX - total) { + /* An overflowing sum cannot match any count. */ + return CMT_FALSE; + } + total += value; + } + + if (monotonic && sample->hist_buckets[count - 1] == sample_count) { + return CMT_FALSE; + } + + return total == sample_count; +} + +static void normalize_histogram_sample(struct cmt_metric *sample, size_t count) +{ + uint64_t total; + size_t index; + + if (!histogram_sample_is_independent(sample, count)) { + return; + } + + total = 0; + for (index = 0; index < count; index++) { + /* No overflow: the sum was validated against count. */ + total += sample->hist_buckets[index]; + sample->hist_buckets[index] = total; + } +} + +int cmt_otlp_histogram_restore(struct cmt *context) +{ + struct cfl_list *head; + struct cfl_list *sample_head; + struct cmt_histogram *histogram; + struct cmt_metric *sample; + + /* A bucket marker is advisory at most: drop it whatever its value. */ + if (cfl_kvlist_fetch(context->internal_metadata, HISTOGRAM_MARKER_KEY) != NULL) { + cfl_kvlist_remove(context->internal_metadata, HISTOGRAM_MARKER_KEY); + } + + cfl_list_foreach(head, &context->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + if (histogram->buckets == NULL) { + continue; + } + + if (histogram->map->metric_static_set) { + normalize_histogram_sample(&histogram->map->metric, + histogram->buckets->count + 1); + } + + cfl_list_foreach(sample_head, &histogram->map->metrics) { + sample = cfl_list_entry(sample_head, struct cmt_metric, _head); + normalize_histogram_sample(sample, histogram->buckets->count + 1); + } + } + + return 0; +} diff --git a/lib/cmetrics/tests/cat.c b/lib/cmetrics/tests/cat.c index 63852e623a0..b2408cb5cc4 100644 --- a/lib/cmetrics/tests/cat.c +++ b/lib/cmetrics/tests/cat.c @@ -28,6 +28,9 @@ #include #include #include +#include + +#include #include "cmt_tests.h" @@ -1083,7 +1086,200 @@ void test_cat_static_labels_empty() cmt_destroy(dst); } +static struct cmt *create_otlp_gauge_context(char *label, double value, uint64_t flags) +{ + struct cmt *cmt; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + + cmt = cmt_create(); + if (cmt == NULL) { + return NULL; + } + + gauge = cmt_gauge_create(cmt, "", "", "otlp_gauge", "otlp gauge", + 1, (char *[]) {"key"}); + if (gauge == NULL || + cmt_gauge_set(gauge, 100, value, 1, (char *[]) {label}) != 0) { + cmt_destroy(cmt); + return NULL; + } + + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata == NULL || + cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", flags) != 0) { + cmt_destroy(cmt); + return NULL; + } + + return cmt; +} + +static uint64_t sample_flags(struct cmt_metric *sample) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(sample->otlp_metadata, "flags"); + if (value == NULL) { + return UINT64_MAX; + } + if (value->type == CFL_VARIANT_UINT) { + return value->data.as_uint64; + } + if (value->type == CFL_VARIANT_INT) { + return (uint64_t) value->data.as_int64; + } + return UINT64_MAX; +} + +/* Samples carrying OTLP metadata must merge by labels, never duplicate. */ +void test_cat_otlp_metadata_samples_merge_by_labels() +{ + int ret; + struct cmt *first; + struct cmt *second; + struct cmt *other; + struct cmt *final; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cmt_map_label *label; + struct cfl_list *head; + int found_v; + int found_w; + + first = create_otlp_gauge_context("v", 1.0, 1); + second = create_otlp_gauge_context("v", 2.0, 2); + other = create_otlp_gauge_context("w", 3.0, 3); + final = cmt_create(); + TEST_ASSERT(first != NULL && second != NULL && other != NULL && final != NULL); + + ret = cmt_cat(final, first); + TEST_CHECK(ret == 0); + ret = cmt_cat(final, second); + TEST_CHECK(ret == 0); + ret = cmt_cat(final, second); + TEST_CHECK(ret == 0); + + TEST_CHECK(cfl_list_size(&final->gauges) == 1); + gauge = cfl_list_entry_first(&final->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 1); + + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_CHECK(cmt_metric_get_value(sample) == 2.0); + TEST_CHECK(sample_flags(sample) == 2); + + /* The merged sample remains reachable through the label index. */ + ret = cmt_gauge_set(gauge, 200, 5.0, 1, (char *[]) {"v"}); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 1); + TEST_CHECK(cmt_metric_get_value(sample) == 5.0); + + /* A distinct label set is still appended. */ + ret = cmt_cat(final, other); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 2); + + found_v = 0; + found_w = 0; + cfl_list_foreach(head, &gauge->map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + label = cfl_list_entry_first(&sample->labels, struct cmt_map_label, _head); + if (strcmp(label->name, "v") == 0) { + found_v++; + TEST_CHECK(sample_flags(sample) == 2); + } + else if (strcmp(label->name, "w") == 0) { + found_w++; + TEST_CHECK(sample_flags(sample) == 3); + TEST_CHECK(cmt_metric_get_value(sample) == 3.0); + } + } + TEST_CHECK(found_v == 1); + TEST_CHECK(found_w == 1); + + cmt_destroy(first); + cmt_destroy(second); + cmt_destroy(other); + cmt_destroy(final); +} + +/* Distinct OTLP data points of one series (different time windows) are kept, + * but concatenating them again must not duplicate either of them. */ +void test_cat_otlp_metadata_distinct_points_kept() +{ + int ret; + int pass; + struct cmt *source; + struct cmt *final; + struct cmt_gauge *gauge; + struct cmt_metric *first; + struct cmt_metric *second; + struct cmt_map_label *label; + struct cfl_list *head; + uint64_t timestamps; + + source = create_otlp_gauge_context("v", 1.0, 1); + TEST_ASSERT(source != NULL); + + /* Add a second data point of the same series at a later time, the way + * the OTLP decoder appends independent points. */ + gauge = cfl_list_entry_first(&source->gauges, struct cmt_gauge, _head); + first = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + second = calloc(1, sizeof(struct cmt_metric)); + TEST_ASSERT(second != NULL); + cfl_list_init(&second->labels); + cfl_list_init(&second->_hash_head); + second->map = gauge->map; + second->hash = first->hash; + cfl_list_add(&second->_head, &gauge->map->metrics); + TEST_ASSERT(cmt_map_metric_get(&gauge->opts, gauge->map, 1, (char *[]) {"v"}, + CMT_FALSE) == first); + label = calloc(1, sizeof(struct cmt_map_label)); + TEST_ASSERT(label != NULL); + label->name = cfl_sds_create("v"); + TEST_ASSERT(label->name != NULL); + cfl_list_add(&label->_head, &second->labels); + cmt_metric_set_double(second, 200, 2.0); + second->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(second->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(second->otlp_metadata, "flags", 2) == 0); + + final = cmt_create(); + TEST_ASSERT(final != NULL); + + for (pass = 0; pass < 2; pass++) { + ret = cmt_cat(final, source); + TEST_CHECK(ret == 0); + + gauge = cfl_list_entry_first(&final->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 2); + + timestamps = 0; + cfl_list_foreach(head, &gauge->map->metrics) { + first = cfl_list_entry(head, struct cmt_metric, _head); + timestamps += cmt_metric_get_timestamp(first); + if (cmt_metric_get_timestamp(first) == 100) { + TEST_CHECK(sample_flags(first) == 1); + TEST_CHECK(cmt_metric_get_value(first) == 1.0); + } + else { + TEST_CHECK(sample_flags(first) == 2); + TEST_CHECK(cmt_metric_get_value(first) == 2.0); + } + } + TEST_CHECK(timestamps == 300); + } + + cmt_destroy(source); + cmt_destroy(final); +} + TEST_LIST = { + {"cat_otlp_metadata_distinct_points_kept", test_cat_otlp_metadata_distinct_points_kept}, + {"cat_otlp_metadata_samples_merge_by_labels", test_cat_otlp_metadata_samples_merge_by_labels}, {"cat_static_labels_source_dynamic_conflict", test_cat_static_labels_source_dynamic_conflict}, {"cat_static_labels_dynamic_conflict", test_cat_static_labels_dynamic_conflict}, {"cat_static_labels_chain", test_cat_static_labels_chain}, diff --git a/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c new file mode 100644 index 00000000000..e598dc275c6 --- /dev/null +++ b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c @@ -0,0 +1,39 @@ +/* + * Generator for histogram-cumulative-v5.1.0.msgpack, a fixture written by the + * bundled CMetrics library of Fluent Bit v5.1.0. It holds the native layout + * that release used for histograms built with cmt_histogram_observe() or + * decoded from Prometheus: cumulative buckets [1, 3, 5, 6], bounds + * [0.1, 0.5, 1] and count 6, once as the static sample and once labeled + * path="/a". The chunk is unmarked; readers infer the layout per sample. + * + * Build it against the headers and static libraries of a v5.1.0 checkout and + * redirect stdout to the fixture. The fixture is versioned on purpose: tests + * must never regenerate it with the current encoder. + */ +#include +#include +#include +#include +#include +int main(void) +{ + struct cmt *ctx = cmt_create(); + struct cmt_histogram *h; + char *keys[] = {"path"}; + char *vals[] = {"/a"}; + double samples[] = {0.05, 0.2, 0.3, 0.7, 0.8, 2.0}; + char *buffer; + size_t size; + int i; + h = cmt_histogram_create(ctx, "", "", "req_seconds", "request latency", + cmt_histogram_buckets_create(3, 0.1, 0.5, 1.0), 1, keys); + for (i = 0; i < 6; i++) { + cmt_histogram_observe(h, 100, samples[i], 1, vals); + cmt_histogram_observe(h, 100, samples[i], 0, NULL); + } + if (cmt_encode_msgpack_create(ctx, &buffer, &size) != 0) { return 1; } + fwrite(buffer, 1, size, stdout); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + return 0; +} diff --git a/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.msgpack b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..7384021aed0a2cb751d6b4f4ca26801f90e6ade9 GIT binary patch literal 343 zcmb7;F;2rk5JgQQQX-B3C!pW}<|4?NtVin=>s_-mjz||`piUqSRF<}QL#Der{J3B1p$@--R(1HZS>8cPhq5VVO|%XV76Z@?__nB+;-^FrTj?Mz twzSz!cegjEJgramk6S$on;Y)=1Skov{%hM-u8yE3e3pmZ)9=Y_?-!*Okb3|C literal 0 HcmV?d00001 diff --git a/lib/cmetrics/tests/data/histogram-v5.1.0.c b/lib/cmetrics/tests/data/histogram-v5.1.0.c new file mode 100644 index 00000000000..2a9d8fc1fd2 --- /dev/null +++ b/lib/cmetrics/tests/data/histogram-v5.1.0.c @@ -0,0 +1,33 @@ +/* + * Generator for histogram-v5.1.0.msgpack, a fixture written by the bundled + * CMetrics library of Fluent Bit v5.1.0. It holds an OTLP-style histogram with + * independent bucket counts [5, 1, 3], count 9, sum 12 and bounds [1, 2]; that + * release stored OTLP-decoded histograms in this representation and wrote no + * bucket-layout marker. + * + * Build it against the headers and static libraries of a v5.1.0 checkout and + * redirect stdout to the fixture. The fixture is versioned on purpose: tests + * must never regenerate it with the current encoder. + */ +#include +#include +#include +#include +#include +int main(void) +{ + struct cmt *ctx = cmt_create(); + struct cmt_histogram *h; + uint64_t buckets[] = {5, 1, 3}; + char *buffer; + size_t size; + h = cmt_histogram_create(ctx, "", "", "legacy_histogram", "legacy", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + h->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + cmt_histogram_set_default(h, 100, buckets, 12.0, 9, 0, NULL); + if (cmt_encode_msgpack_create(ctx, &buffer, &size) != 0) { return 1; } + fwrite(buffer, 1, size, stdout); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + return 0; +} diff --git a/lib/cmetrics/tests/data/histogram-v5.1.0.msgpack b/lib/cmetrics/tests/data/histogram-v5.1.0.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..014606de3a12dd4fbfd1a76a5aeb63401f2f5cd2 GIT binary patch literal 250 zcmX}lORmB&3d-R@q&}4K2Dcuud rWlj8g)v6|-J<=vGnMUWIo*zHRuG{MD$*1+8-xvpBz2RO+fRg9|nh$49 literal 0 HcmV?d00001 diff --git a/lib/cmetrics/tests/msgpack_abi.c b/lib/cmetrics/tests/msgpack_abi.c index c9d5d13e318..d35fd47159e 100644 --- a/lib/cmetrics/tests/msgpack_abi.c +++ b/lib/cmetrics/tests/msgpack_abi.c @@ -347,7 +347,228 @@ void test_msgpack_abi_histogram_buckets_reject_mismatch() cmt_destroy(cmt); } +static struct cmt *create_static_gauge_with_metadata(void) +{ + struct cmt *source; + struct cmt_gauge *gauge; + struct cfl_kvlist *metadata; + + source = cmt_create(); + if (source == NULL) { + return NULL; + } + + gauge = cmt_gauge_create(source, "", "", "windows", "windows", 0, NULL); + if (gauge == NULL || cmt_gauge_set(gauge, 1, 1.0, 0, NULL) != 0) { + cmt_destroy(source); + return NULL; + } + + metadata = cfl_kvlist_create(); + gauge->map->metric.otlp_metadata = metadata; + if (metadata == NULL || + cfl_kvlist_insert_uint64(metadata, "flags", 7) != 0 || + cfl_kvlist_insert_string(metadata, "text", "value") != 0 || + cfl_kvlist_insert_bytes(metadata, "binary", "\0\xff", 2, CFL_FALSE) != 0) { + cmt_destroy(source); + return NULL; + } + + return source; +} + +static void test_point_metadata_and_unlabeled_windows(void) +{ + struct cmt *source; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *result; + struct cmt_metric *point; + struct cfl_variant *value; + char *buffer; + size_t size; + size_t offset; + int ret; + + /* Typed static point metadata survives a round trip. */ + source = create_static_gauge_with_metadata(); + TEST_ASSERT(source != NULL); + TEST_CHECK(cmt_encode_msgpack_create(source, &buffer, &size) == 0); + offset = 0; + ret = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_ASSERT(ret == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(offset == size); + result = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(result != NULL); + TEST_CHECK(cmt_metric_get_timestamp(&result->map->metric) == 1); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "flags"); + TEST_ASSERT(value != NULL); + TEST_CHECK((value->type == CFL_VARIANT_UINT && value->data.as_uint64 == 7) || + (value->type == CFL_VARIANT_INT && value->data.as_int64 == 7)); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "text"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->type == CFL_VARIANT_STRING && cfl_variant_size_get(value) == 5); + TEST_CHECK(memcmp(value->data.as_string, "value", 5) == 0); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "binary"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->type == CFL_VARIANT_BYTES && cfl_variant_size_get(value) == 2); + TEST_CHECK(memcmp(value->data.as_bytes, "\0\xff", 2) == 0); + TEST_CHECK(cfl_list_size(&result->map->metrics) == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(source); + + /* A further label-less point is the same series: like released readers, + * it replaces the static metric (last one wins) instead of creating a + * duplicate series. Its point metadata no longer lines up with the + * sidecar and is skipped, while the values and the chunk survive. */ + source = create_static_gauge_with_metadata(); + TEST_ASSERT(source != NULL); + gauge = cfl_list_entry_first(&source->gauges, struct cmt_gauge, _head); + point = calloc(1, sizeof(*point)); + TEST_ASSERT(point != NULL); + cfl_list_init(&point->labels); + point->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(point->otlp_metadata != NULL); + TEST_CHECK(cfl_kvlist_insert_uint64(point->otlp_metadata, "flags", 3) == 0); + cmt_metric_set_double(point, 2, 2.0); + cfl_list_add(&point->_head, &gauge->map->metrics); + TEST_CHECK(cmt_encode_msgpack_create(source, &buffer, &size) == 0); + offset = 0; + ret = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_ASSERT(ret == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(offset == size); + result = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(result != NULL); + TEST_CHECK(cfl_list_size(&result->map->metrics) == 0); + TEST_CHECK(cmt_metric_get_timestamp(&result->map->metric) == 2); + TEST_CHECK(cmt_metric_get_value(&result->map->metric) == 2.0); + TEST_CHECK(result->map->metric.otlp_metadata == NULL); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(source); +} + +static void test_index_resize_preserves_family_metadata(void) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cfl_variant *value; + char label[32]; + char *values[] = {label}; + char *keys[] = {"series"}; + size_t index; + char *buffer; + size_t size; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "resize", "resize", 1, keys); + TEST_ASSERT(gauge != NULL); + gauge->map->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(gauge->map->otlp_metadata, "family", "retained") == 0); + for (index = 0; index < 300; index++) { + snprintf(label, sizeof(label), "%zu", index); + TEST_ASSERT(cmt_gauge_set(gauge, 100, index, 1, values) == 0); + } + TEST_CHECK(gauge->map->metric_bucket_count > 64); + value = cfl_kvlist_fetch(gauge->map->otlp_metadata, "family"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_STRING); + if (value != NULL && value->type == CFL_VARIANT_STRING) { + TEST_CHECK(strcmp(value->data.as_string, "retained") == 0); + } + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void test_stale_metadata_sidecar_does_not_drop_chunk(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_array *sidecar; + char *buffer; + size_t size; + size_t offset = 0; + int result; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "retained", "retained", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + /* An intermediary removed a family without updating the optional sidecar. */ + sidecar = cfl_array_create(0); + TEST_ASSERT(sidecar != NULL); + TEST_ASSERT(cfl_kvlist_insert_array(ctx->internal_metadata, + "__cmetrics_native_metadata_v1", sidecar) == 0); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + if (decoded != NULL) { + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 42); + TEST_CHECK(cfl_kvlist_fetch(decoded->internal_metadata, + "__cmetrics_native_metadata_v1") == NULL); + cmt_destroy(decoded); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void test_invalid_optional_metadata_does_not_drop_chunk(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_array *sidecar; + struct cfl_array *points; + struct cfl_kvlist *family; + char *buffer; + size_t size; + size_t offset = 0; + int result; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "retained", "retained", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + sidecar = cfl_array_create(1); + points = cfl_array_create(1); + family = cfl_kvlist_create(); + TEST_ASSERT(sidecar != NULL && points != NULL && family != NULL); + TEST_ASSERT(cfl_array_append_null(points) == 0); + TEST_ASSERT(cfl_kvlist_insert_array(family, "points", points) == 0); + TEST_ASSERT(cfl_kvlist_insert_bool(family, "allow_reset", CFL_FALSE) == 0); + TEST_ASSERT(cfl_kvlist_insert_bytes(family, "family", "\xc1", 1, CFL_FALSE) == 0); + TEST_ASSERT(cfl_array_append_kvlist(sidecar, family) == 0); + TEST_ASSERT(cfl_kvlist_insert_array(ctx->internal_metadata, + "__cmetrics_native_metadata_v1", sidecar) == 0); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + if (decoded != NULL) { + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 42); + TEST_CHECK(gauge->map->otlp_metadata == NULL); + cmt_destroy(decoded); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + TEST_LIST = { + {"invalid_optional_metadata_retains_chunk", test_invalid_optional_metadata_does_not_drop_chunk}, + {"stale_metadata_sidecar_does_not_drop_chunk", test_stale_metadata_sidecar_does_not_drop_chunk}, + {"index_resize_preserves_family_metadata", test_index_resize_preserves_family_metadata}, + {"point_metadata_and_unlabeled_windows", test_point_metadata_and_unlabeled_windows}, {"msgpack_abi_legacy_value_only_decode", test_msgpack_abi_legacy_value_only_decode}, {"msgpack_abi_typed_int64_decode", test_msgpack_abi_typed_int64_decode}, {"msgpack_abi_summary_quantiles_reject_mismatch", test_msgpack_abi_summary_quantiles_reject_mismatch}, diff --git a/lib/cmetrics/tests/msgpack_temporality.c b/lib/cmetrics/tests/msgpack_temporality.c index b4a36e94aa0..e161aa88ed0 100644 --- a/lib/cmetrics/tests/msgpack_temporality.c +++ b/lib/cmetrics/tests/msgpack_temporality.c @@ -37,6 +37,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -174,6 +178,7 @@ static void test_counter_temporality_still_survives_msgpack_roundtrip() TEST_CHECK(counter != NULL); counter->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + counter->allow_reset = CMT_TRUE; offset = 0; result = cmt_encode_msgpack_create(cmt, &encoded_buffer, &encoded_size); @@ -191,6 +196,7 @@ static void test_counter_temporality_still_survives_msgpack_roundtrip() TEST_CHECK(decoded_counter != NULL); if (decoded_counter != NULL) { + TEST_CHECK(decoded_counter->allow_reset == CMT_TRUE); TEST_CHECK(decoded_counter->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); } @@ -202,7 +208,295 @@ static void test_counter_temporality_still_survives_msgpack_roundtrip() cmt_destroy(cmt); } +static void test_legacy_histogram_bucket_semantics() +{ + cfl_sds_t fixture; + struct cmt *decoded; + struct cmt_histogram *histogram; + size_t offset = 0; + + fixture = read_file(CMT_TESTS_DATA_PATH "/histogram-v5.1.0.msgpack"); + TEST_ASSERT(fixture != NULL); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, fixture, + cfl_sds_len(fixture), &offset) == 0); + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + /* Legacy OTLP chunks contain independent bucket counts [5,1,3]. */ + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 0) == 5); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 1) == 6); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 2) == 9); + TEST_CHECK(offset == cfl_sds_len(fixture)); + cmt_destroy(decoded); + cfl_sds_destroy(fixture); +} + +static int buffer_contains(char *buffer, size_t size, const void *needle, size_t length) +{ + size_t index; + + for (index = 0; index + length <= size; index++) { + if (memcmp(buffer + index, needle, length) == 0) { + return CMT_TRUE; + } + } + + return CMT_FALSE; +} + +static void check_histogram_sample(struct cmt_metric *sample, uint64_t *expected, + size_t count) +{ + size_t index; + + for (index = 0; index < count; index++) { + TEST_CHECK(cmt_metric_hist_get_value(sample, index) == expected[index]); + TEST_MSG("bucket %zu: expected %" PRIu64 ", got %" PRIu64, index, + expected[index], cmt_metric_hist_get_value(sample, index)); + } +} + +/* Fluent Bit v5.1.0 natively writes cumulative (Prometheus-style) buckets. */ +static void test_legacy_cumulative_histogram_fixture() +{ + cfl_sds_t fixture; + struct cmt *decoded; + struct cmt_histogram *histogram; + struct cmt_metric *sample; + size_t offset = 0; + uint64_t cumulative[] = {1, 3, 5, 6}; + uint64_t independent[] = {1, 2, 2, 1}; + size_t index; + + fixture = read_file(CMT_TESTS_DATA_PATH "/histogram-cumulative-v5.1.0.msgpack"); + TEST_ASSERT(fixture != NULL); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, fixture, + cfl_sds_len(fixture), &offset) == 0); + TEST_CHECK(offset == cfl_sds_len(fixture)); + + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + TEST_ASSERT(histogram != NULL); + TEST_CHECK(histogram->map->metric_static_set == CMT_TRUE); + check_histogram_sample(&histogram->map->metric, cumulative, 4); + + TEST_ASSERT(cfl_list_size(&histogram->map->metrics) == 1); + sample = cfl_list_entry_first(&histogram->map->metrics, struct cmt_metric, _head); + check_histogram_sample(sample, cumulative, 4); + TEST_CHECK(cmt_metric_hist_get_count_value(sample) == 6); + + for (index = 0; index < 4; index++) { + TEST_CHECK(cmt_metric_hist_get_bucket_count(sample, index) == independent[index]); + } + + cmt_destroy(decoded); + cfl_sds_destroy(fixture); +} + +/* The wire must stay byte-compatible with v5.1.0 readers: cumulative and + * unmarked, with or without the OTLP metadata sidecar. */ +static void test_histogram_wire_stays_cumulative() +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_histogram *histogram; + char *buffer; + size_t size; + size_t offset; + int pass; + uint64_t counts[] = {5, 6, 9}; + const unsigned char expected[] = {0xa7, 'b','u','c','k','e','t','s',0x93,5,6,9}; + const char marker[] = "__cmetrics_histogram_buckets"; + + for (pass = 0; pass < 2; pass++) { + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + histogram = cmt_histogram_create(ctx, "", "", "hist", "hist", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_set_default(histogram, 100, counts, 12.0, 9, 0, NULL) == 0); + + if (pass == 1) { + /* Force the OTLP metadata sidecar path. */ + histogram->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(histogram->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(histogram->map->metric.otlp_metadata, + "flags", 1) == 0); + } + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + TEST_CHECK(buffer_contains(buffer, size, expected, sizeof(expected))); + TEST_CHECK(!buffer_contains(buffer, size, marker, sizeof(marker) - 1)); + + offset = 0; + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + check_histogram_sample(&histogram->map->metric, counts, 3); + if (pass == 1) { + TEST_CHECK(histogram->map->metric.otlp_metadata != NULL); + } + + cmt_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + } +} + +/* Encode a histogram and splice a bucket marker into the internal metadata + * map, as unreleased writers did; the current encoder never emits it. */ +static int encode_with_marker(uint64_t *counts, uint64_t total, + const char *marker, size_t marker_length, + char **buffer, size_t *size) +{ + struct cmt *ctx; + struct cmt_histogram *histogram; + char *encoded; + char *spliced; + size_t encoded_size; + size_t index; + size_t position; + size_t key_length; + const char header[] = "\xa8" "cmetrics" "\x80"; + const char key[] = "__cmetrics_histogram_buckets"; + + ctx = cmt_create(); + if (ctx == NULL) { + return -1; + } + + histogram = cmt_histogram_create(ctx, "", "", "hist", "hist", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + if (histogram == NULL || + cmt_histogram_set_default(histogram, 100, counts, 12.0, total, 0, NULL) != 0 || + cmt_encode_msgpack_create(ctx, &encoded, &encoded_size) != 0) { + cmt_destroy(ctx); + return -1; + } + cmt_destroy(ctx); + + position = encoded_size; + for (index = 0; index + sizeof(header) - 1 <= encoded_size; index++) { + if (memcmp(encoded + index, header, sizeof(header) - 1) == 0) { + position = index + sizeof(header) - 2; + break; + } + } + if (position == encoded_size) { + cmt_encode_msgpack_destroy(encoded); + return -1; + } + + key_length = sizeof(key) - 1; + spliced = malloc(encoded_size + 2 + key_length + marker_length); + if (spliced == NULL) { + cmt_encode_msgpack_destroy(encoded); + return -1; + } + + /* fixmap(1) { fixstr(key): marker } replaces the empty fixmap. */ + memcpy(spliced, encoded, position); + index = position; + spliced[index++] = (char) 0x81; + spliced[index++] = (char) (0xa0 | key_length); + memcpy(spliced + index, key, key_length); + index += key_length; + memcpy(spliced + index, marker, marker_length); + index += marker_length; + memcpy(spliced + index, encoded + position + 1, encoded_size - position - 1); + index += encoded_size - position - 1; + + cmt_encode_msgpack_destroy(encoded); + *buffer = spliced; + *size = index; + + return 0; +} + +/* Bucket markers written by unreleased builds are advisory: unknown or + * mistyped values must not drop the chunk, and a marker can not force a + * wrong interpretation of the buckets. */ +static void test_histogram_marker_is_ignored() +{ + struct cmt *decoded; + struct cmt_histogram *histogram; + char *buffer; + size_t size; + size_t offset; + size_t index; + int ret; + uint64_t cumulative[] = {5, 6, 9}; + uint64_t independent[] = {5, 1, 3}; + struct { + const char *marker; + size_t length; + uint64_t *counts; + } cases[] = { + {"\xa5" "bogus", 6, cumulative}, + {"\x07", 1, cumulative}, + {"\xc0", 1, cumulative}, + {"\xaa" "per_bucket", 11, cumulative}, + /* spoofed: "cumulative" claimed on independent counts */ + {"\xaa" "cumulative", 11, independent}, + {"\xaa" "cumulative", 11, cumulative}, + {"\xaa" "per_bucket", 11, independent}, + }; + + for (index = 0; index < sizeof(cases) / sizeof(cases[0]); index++) { + buffer = NULL; + size = 0; + ret = encode_with_marker(cases[index].counts, 9, cases[index].marker, + cases[index].length, &buffer, &size); + TEST_ASSERT(ret == 0); + + offset = 0; + decoded = NULL; + ret = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(ret == CMT_DECODE_MSGPACK_SUCCESS); + TEST_MSG("case %zu: decode returned %d", index, ret); + TEST_CHECK(offset == size); + + if (decoded != NULL) { + histogram = cfl_list_entry_first(&decoded->histograms, + struct cmt_histogram, _head); + check_histogram_sample(&histogram->map->metric, cumulative, 3); + TEST_CHECK(cfl_kvlist_fetch(decoded->internal_metadata, + "__cmetrics_histogram_buckets") == NULL); + cmt_destroy(decoded); + } + + free(buffer); + } +} + +/* Samples whose shape is not consistent with either layout are untouched. */ +static void test_histogram_inconsistent_buckets_untouched() +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_histogram *histogram; + char *buffer; + size_t size; + size_t offset = 0; + uint64_t counts[] = {4, 2, 8}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + histogram = cmt_histogram_create(ctx, "", "", "hist", "hist", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_set_default(histogram, 100, counts, 1.0, 11, 0, NULL) == 0); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + check_histogram_sample(&histogram->map->metric, counts, 3); + cmt_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + TEST_LIST = { + {"legacy_histogram_bucket_semantics", test_legacy_histogram_bucket_semantics}, + {"legacy_cumulative_histogram_fixture", test_legacy_cumulative_histogram_fixture}, + {"histogram_wire_stays_cumulative", test_histogram_wire_stays_cumulative}, + {"histogram_marker_is_ignored", test_histogram_marker_is_ignored}, + {"histogram_inconsistent_buckets_untouched", test_histogram_inconsistent_buckets_untouched}, {"histogram_temporality_survives_msgpack_roundtrip", test_histogram_temporality_survives_msgpack_roundtrip}, {"exp_histogram_temporality_survives_msgpack_roundtrip", diff --git a/lib/cmetrics/tests/opentelemetry.c b/lib/cmetrics/tests/opentelemetry.c index f225a7fcfb9..996df4049f9 100644 --- a/lib/cmetrics/tests/opentelemetry.c +++ b/lib/cmetrics/tests/opentelemetry.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -1856,11 +1857,8 @@ static void test_opentelemetry_zero_length_bytes_attribute(void) attribute = roundtrip_dp->attributes[0]; TEST_CHECK(strcmp(attribute->key, "empty_bytes") == 0); TEST_CHECK(attribute->value->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE); - TEST_CHECK(attribute->value->string_value != NULL); - if (attribute->value->string_value != NULL) { - TEST_CHECK(attribute->value->string_value[0] == '\0'); - } + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE); + TEST_CHECK(attribute->value->bytes_value.len == 0); } } } @@ -2813,7 +2811,214 @@ static void test_opentelemetry_summary_layout_mismatch(void) cfl_sds_destroy(payload); } +static void test_otlp_unsigned_attribute_preserves_signed_range(void) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cfl_kvlist *attrs; + cfl_sds_t wire; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *point; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "uint_attributes", "uint attributes", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + gauge->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->metric.otlp_metadata != NULL); + attrs = cfl_kvlist_create(); + TEST_ASSERT(attrs != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(attrs, "small", 3) == 0); + TEST_ASSERT(cfl_kvlist_insert_uint64(attrs, "maximum", INT64_MAX) == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->metric.otlp_metadata, "attributes", attrs) == 0); + TEST_ASSERT(cmt_otlp_capture_labels(gauge->map, &gauge->map->metric) == 0); + wire = cmt_encode_opentelemetry_create(ctx); + TEST_CHECK(wire != NULL); + if (wire != NULL) { + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + point = request->resource_metrics[0]->scope_metrics[0]->metrics[0]->gauge->data_points[0]; + TEST_ASSERT(point->n_attributes == 2); + TEST_CHECK(point->attributes[0]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE); + TEST_CHECK(point->attributes[0]->value->int_value == 3); + TEST_CHECK(point->attributes[1]->value->int_value == INT64_MAX); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + } + cmt_destroy(ctx); +} + +/* Repeated label-less points of one metric are the same series: the last one + * replaces the static sample, as released versions behave. */ +static cfl_sds_t generate_repeated_unlabeled_points_payload() +{ + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest request; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics resource_metrics; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics scope_metrics; + Opentelemetry__Proto__Metrics__V1__Metric gauge_metric; + Opentelemetry__Proto__Metrics__V1__Metric histogram_metric; + Opentelemetry__Proto__Metrics__V1__Gauge gauge; + Opentelemetry__Proto__Metrics__V1__Histogram histogram; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint number_points[2]; + Opentelemetry__Proto__Metrics__V1__HistogramDataPoint histogram_points[2]; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics *resource_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics *scope_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__Metric *metric_list[2]; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *number_point_list[2]; + Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *histogram_point_list[2]; + uint64_t first_buckets[] = {5, 1, 3}; + uint64_t second_buckets[] = {1, 1, 1}; + double bounds[] = {1.0, 2.0}; + size_t payload_size; + size_t index; + unsigned char *packed_payload; + cfl_sds_t payload; + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(&request); + opentelemetry__proto__metrics__v1__resource_metrics__init(&resource_metrics); + opentelemetry__proto__metrics__v1__scope_metrics__init(&scope_metrics); + opentelemetry__proto__metrics__v1__metric__init(&gauge_metric); + opentelemetry__proto__metrics__v1__metric__init(&histogram_metric); + opentelemetry__proto__metrics__v1__gauge__init(&gauge); + opentelemetry__proto__metrics__v1__histogram__init(&histogram); + + for (index = 0; index < 2; index++) { + opentelemetry__proto__metrics__v1__number_data_point__init(&number_points[index]); + number_points[index].time_unix_nano = index + 1; + number_points[index].flags = index; + number_points[index].value_case = + OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + number_points[index].as_double = index + 1.0; + number_point_list[index] = &number_points[index]; + + opentelemetry__proto__metrics__v1__histogram_data_point__init(&histogram_points[index]); + histogram_points[index].time_unix_nano = index + 1; + histogram_points[index].n_bucket_counts = 3; + histogram_points[index].bucket_counts = index == 0 ? first_buckets : second_buckets; + histogram_points[index].n_explicit_bounds = 2; + histogram_points[index].explicit_bounds = bounds; + histogram_points[index].count = index == 0 ? 9 : 3; + histogram_points[index].has_sum = 1; + histogram_points[index].sum = 1.0; + histogram_point_list[index] = &histogram_points[index]; + } + + gauge.n_data_points = 2; + gauge.data_points = number_point_list; + gauge_metric.name = "u"; + gauge_metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE; + gauge_metric.gauge = &gauge; + + histogram.n_data_points = 2; + histogram.data_points = histogram_point_list; + histogram.aggregation_temporality = + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE; + histogram_metric.name = "h"; + histogram_metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM; + histogram_metric.histogram = &histogram; + + metric_list[0] = &gauge_metric; + metric_list[1] = &histogram_metric; + scope_metrics.n_metrics = 2; + scope_metrics.metrics = metric_list; + + scope_metrics_list[0] = &scope_metrics; + resource_metrics.n_scope_metrics = 1; + resource_metrics.scope_metrics = scope_metrics_list; + + resource_metrics_list[0] = &resource_metrics; + request.n_resource_metrics = 1; + request.resource_metrics = resource_metrics_list; + + payload_size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(&request); + packed_payload = calloc(1, payload_size); + if (packed_payload == NULL) { + return NULL; + } + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(&request, + packed_payload); + + payload = cfl_sds_create_len((char *) packed_payload, payload_size); + free(packed_payload); + + return payload; +} + +static void test_opentelemetry_repeated_unlabeled_points_last_wins(void) +{ + cfl_sds_t payload; + cfl_sds_t encoded; + size_t offset; + int result; + struct cfl_list decoded_context_list; + struct cmt *decoded_context; + struct cmt_gauge *gauge; + struct cmt_histogram *histogram; + struct cfl_variant *flags; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + payload = generate_repeated_unlabeled_points_payload(); + TEST_ASSERT(payload != NULL); + + offset = 0; + result = cmt_decode_opentelemetry_create(&decoded_context_list, payload, + cfl_sds_len(payload), &offset); + TEST_ASSERT(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + + decoded_context = cfl_list_entry_first(&decoded_context_list, struct cmt, _head); + TEST_ASSERT(decoded_context != NULL); + + gauge = cfl_list_entry_first(&decoded_context->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 0); + TEST_CHECK(gauge->map->metric_static_set == CMT_TRUE); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 2.0); + TEST_CHECK(cmt_metric_get_timestamp(&gauge->map->metric) == 2); + flags = cfl_kvlist_fetch(gauge->map->metric.otlp_metadata, "flags"); + TEST_ASSERT(flags != NULL); + TEST_CHECK(flags->type == CFL_VARIANT_UINT && flags->data.as_uint64 == 1); + + histogram = cfl_list_entry_first(&decoded_context->histograms, struct cmt_histogram, _head); + TEST_ASSERT(histogram != NULL); + TEST_CHECK(cfl_list_size(&histogram->map->metrics) == 0); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 0) == 1); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 1) == 2); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 2) == 3); + TEST_CHECK(cmt_metric_hist_get_count_value(&histogram->map->metric) == 3); + + encoded = cmt_encode_opentelemetry_create(decoded_context); + TEST_ASSERT(encoded != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(encoded), (const uint8_t *) encoded); + TEST_ASSERT(request != NULL); + + metric = find_otlp_metric_by_name(request, "u"); + TEST_ASSERT(metric != NULL && metric->gauge != NULL); + TEST_CHECK(metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->as_double == 2.0); + + metric = find_otlp_metric_by_name(request, "h"); + TEST_ASSERT(metric != NULL && metric->histogram != NULL); + TEST_CHECK(metric->histogram->n_data_points == 1); + TEST_CHECK(metric->histogram->data_points[0]->n_bucket_counts == 3); + TEST_CHECK(metric->histogram->data_points[0]->bucket_counts[0] == 1); + TEST_CHECK(metric->histogram->data_points[0]->bucket_counts[1] == 1); + TEST_CHECK(metric->histogram->data_points[0]->bucket_counts[2] == 1); + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(encoded); + cmt_decode_opentelemetry_destroy(&decoded_context_list); + cfl_sds_destroy(payload); +} + TEST_LIST = { + {"repeated_unlabeled_points_last_wins", test_opentelemetry_repeated_unlabeled_points_last_wins}, + {"otlp_unsigned_attribute_signed_range", test_otlp_unsigned_attribute_preserves_signed_range}, {"opentelemetry_api_full_roundtrip_with_msgpack", test_opentelemetry_api_full_roundtrip_with_msgpack}, {"opentelemetry_encode_multi_resource_scope_containers", test_opentelemetry_encode_multi_resource_scope_containers}, {"opentelemetry_encode_scope_attributes_cleanup", test_opentelemetry_encode_scope_attributes_cleanup}, From f5f21601990d67b08b840964946e6aab1e9017ba Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 02/61] lib: ctraces: preserve unknown span kinds Keep proto3 enum values received on the wire instead of rejecting the span, matching protobuf decoding semantics. Signed-off-by: Eduardo Silva --- lib/ctraces/src/ctr_decode_msgpack.c | 5 ++- lib/ctraces/src/ctr_decode_opentelemetry.c | 6 +-- lib/ctraces/src/ctr_encode_msgpack.c | 2 +- lib/ctraces/src/ctr_encode_opentelemetry.c | 22 +---------- lib/ctraces/tests/decoding.c | 45 ++++++++++++++++++++++ lib/ctraces/tests/opentelemetry.c | 12 ++++-- 6 files changed, 61 insertions(+), 31 deletions(-) diff --git a/lib/ctraces/src/ctr_decode_msgpack.c b/lib/ctraces/src/ctr_decode_msgpack.c index a77b932ca27..fc3de4a65bb 100644 --- a/lib/ctraces/src/ctr_decode_msgpack.c +++ b/lib/ctraces/src/ctr_decode_msgpack.c @@ -512,8 +512,9 @@ static int unpack_span_kind(mpack_reader_t *reader, size_t index, void *ctx) int result; result = ctr_mpack_consume_int32_tag(reader, &kind); - if (result == CTR_MPACK_SUCCESS && ctr_span_kind_set(context->span, kind) != 0) { - return CTR_MPACK_CORRUPT_INPUT_DATA_ERROR; + if (result == CTR_MPACK_SUCCESS) { + /* Match proto3 decoding: preserve unknown received enum values. */ + context->span->kind = kind; } return result; diff --git a/lib/ctraces/src/ctr_decode_opentelemetry.c b/lib/ctraces/src/ctr_decode_opentelemetry.c index 60ae7b07e29..8bb878e2ad0 100644 --- a/lib/ctraces/src/ctr_decode_opentelemetry.c +++ b/lib/ctraces/src/ctr_decode_opentelemetry.c @@ -774,10 +774,8 @@ int ctr_decode_opentelemetry_create(struct ctrace **out_ctr, ctr_span_set_trace_state(span, otel_span->trace_state, strlen(otel_span->trace_state)); } - if (ctr_span_kind_set(span, otel_span->kind) != 0) { - return decode_opentelemetry_error( - service_request, ctr, CTR_DECODE_OPENTELEMETRY_INVALID_PAYLOAD); - } + /* Preserve unknown proto3 enum values received on the wire. */ + span->kind = otel_span->kind; ctr_span_set_flags(span, otel_span->flags); ctr_span_start_ts(ctr, span, otel_span->start_time_unix_nano); ctr_span_end_ts(ctr, span, otel_span->end_time_unix_nano); diff --git a/lib/ctraces/src/ctr_encode_msgpack.c b/lib/ctraces/src/ctr_encode_msgpack.c index b20b3b00c4e..43f11b033ba 100644 --- a/lib/ctraces/src/ctr_encode_msgpack.c +++ b/lib/ctraces/src/ctr_encode_msgpack.c @@ -363,7 +363,7 @@ static void pack_span(mpack_writer_t *writer, struct ctrace_span *span) /* kind */ mpack_write_cstr(writer, "kind"); - mpack_write_u32(writer, span->kind); + mpack_write_i32(writer, span->kind); /* start_time_unix_nano */ mpack_write_cstr(writer, "start_time_unix_nano"); diff --git a/lib/ctraces/src/ctr_encode_opentelemetry.c b/lib/ctraces/src/ctr_encode_opentelemetry.c index e56770fd089..5c6a114353d 100644 --- a/lib/ctraces/src/ctr_encode_opentelemetry.c +++ b/lib/ctraces/src/ctr_encode_opentelemetry.c @@ -658,26 +658,8 @@ static void otel_span_set_parent_span_id(Opentelemetry__Proto__Trace__V1__Span * static void otel_span_set_kind(Opentelemetry__Proto__Trace__V1__Span *otel_span, int kind) { - switch (kind) { - case CTRACE_SPAN_INTERNAL: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_INTERNAL; - break; - case CTRACE_SPAN_SERVER: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_SERVER; - break; - case CTRACE_SPAN_CLIENT: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_CLIENT; - break; - case CTRACE_SPAN_PRODUCER: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_PRODUCER; - break; - case CTRACE_SPAN_CONSUMER: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_CONSUMER; - break; - default: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_UNSPECIFIED; - break; - } + /* CTrace kind values match the OTLP enum. Preserve unknown proto3 values. */ + otel_span->kind = kind; } static void otel_span_set_start_time(Opentelemetry__Proto__Trace__V1__Span *span, diff --git a/lib/ctraces/tests/decoding.c b/lib/ctraces/tests/decoding.c index 04239b9245c..bee8046600e 100644 --- a/lib/ctraces/tests/decoding.c +++ b/lib/ctraces/tests/decoding.c @@ -625,6 +625,50 @@ void test_msgpack_preserves_flags() ctr_destroy(ctx); } +void test_msgpack_preserves_signed_kind() +{ + char *buffer; + size_t size; + size_t offset; + struct ctrace *ctx; + struct ctrace *decoded; + struct ctrace_resource_span *rs; + struct ctrace_scope_span *ss; + struct ctrace_span *span; + int kinds[] = {-1, INT32_MIN, 0, 5, INT32_MAX}; + size_t index; + + for (index = 0; index < sizeof(kinds) / sizeof(kinds[0]); index++) { + ctx = ctr_create(NULL); + TEST_ASSERT(ctx != NULL); + rs = ctr_resource_span_create(ctx); + TEST_ASSERT(rs != NULL); + ss = ctr_scope_span_create(rs); + TEST_ASSERT(ss != NULL); + span = ctr_span_create(ctx, ss, "unknown-kind", NULL); + TEST_ASSERT(span != NULL); + /* Decoders preserve unknown protobuf enum values. */ + span->kind = kinds[index]; + TEST_ASSERT(ctr_span_create(ctx, ss, "healthy-sibling", NULL) != NULL); + TEST_ASSERT(ctr_encode_msgpack_create(ctx, &buffer, &size) == 0); + offset = 0; + decoded = NULL; + TEST_CHECK(ctr_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + if (decoded != NULL) { + rs = cfl_list_entry(decoded->resource_spans.next, + struct ctrace_resource_span, _head); + ss = cfl_list_entry(rs->scope_spans.next, struct ctrace_scope_span, _head); + span = cfl_list_entry(ss->spans.next, struct ctrace_span, _head); + TEST_CHECK(span->kind == kinds[index]); + TEST_CHECK(cfl_list_size(&ss->spans) == 2); + TEST_CHECK(offset == size); + ctr_destroy(decoded); + } + ctr_encode_msgpack_destroy(buffer); + ctr_destroy(ctx); + } +} + void test_msgpack_invalid_offset() { char data[] = {0x80}; @@ -1013,6 +1057,7 @@ void test_simple_to_msgpack_and_back() TEST_LIST = { + {"msgpack_preserves_signed_kind", test_msgpack_preserves_signed_kind}, {"cmt_simple_to_msgpack_and_back", test_simple_to_msgpack_and_back}, {"cmt_msgpack", test_msgpack_to_cmt}, {"empty_spans", test_msgpack_to_ctr_with_empty_spans}, diff --git a/lib/ctraces/tests/opentelemetry.c b/lib/ctraces/tests/opentelemetry.c index fee972b3982..1d8c249ae7b 100644 --- a/lib/ctraces/tests/opentelemetry.c +++ b/lib/ctraces/tests/opentelemetry.c @@ -670,7 +670,7 @@ void test_otlp_decode_span_empty_name() free(wire); } -void test_otlp_decode_invalid_kind() +void test_otlp_preserves_unknown_kind() { Opentelemetry__Proto__Trace__V1__Span span; uint8_t *wire; @@ -688,8 +688,12 @@ void test_otlp_decode_invalid_kind() offset = 0; decoded = NULL; ret = ctr_decode_opentelemetry_create(&decoded, (char *) wire, wire_len, &offset); - TEST_CHECK(ret != CTR_DECODE_OPENTELEMETRY_SUCCESS); - TEST_CHECK(decoded == NULL); + TEST_ASSERT(ret == CTR_DECODE_OPENTELEMETRY_SUCCESS); + TEST_ASSERT(decoded != NULL); + TEST_CHECK(cfl_list_entry_first(&decoded->span_list, struct ctrace_span, _head_global)->kind == 99); + TEST_CHECK(ctr_span_kind_set(cfl_list_entry_first(&decoded->span_list, + struct ctrace_span, _head_global), 99) == -1); + ctr_destroy(decoded); free(wire); } @@ -774,6 +778,6 @@ TEST_LIST = { {"otlp_decode_attribute_null_value", test_otlp_decode_attribute_null_value}, {"otlp_decode_nested_malformed_kv", test_otlp_decode_nested_malformed_kv}, {"otlp_decode_span_empty_name", test_otlp_decode_span_empty_name}, - {"otlp_decode_invalid_kind", test_otlp_decode_invalid_kind}, + {"otlp_preserves_unknown_kind", test_otlp_preserves_unknown_kind}, { 0 } }; From e2fd81b13440ef50dffb29a8550b9ba3f5d064e7 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 03/61] mp: add record pooling and a log record scanner Pool chunk record wrappers, add an opt-in value arena for chunks that are only serialized, and add an allocation-free scanner that counts log records and detects plain logs. A failed chunk encode now resets the encoder so partial records never leak into the next chunk. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_mp.h | 6 + include/fluent-bit/flb_mp_chunk.h | 19 + src/flb_mp.c | 801 +++++++++++++++++++++--------- 3 files changed, 585 insertions(+), 241 deletions(-) diff --git a/include/fluent-bit/flb_mp.h b/include/fluent-bit/flb_mp.h index 20a5ddcc6d4..831f1666bfa 100644 --- a/include/fluent-bit/flb_mp.h +++ b/include/fluent-bit/flb_mp.h @@ -27,6 +27,12 @@ #define FLB_MP_ARRAY MSGPACK_OBJECT_ARRAY int flb_mp_count(const void *data, size_t bytes); + +/* + * Allocation-free in non-tracking builds; rejects unsupported/deep shapes. + * 'typed' requires string map keys and CFL-compatible value types/depth. + */ +int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *records, int *plain); int flb_mp_count_log_records(const void *data, size_t bytes); int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_size, char **out_buf, size_t *out_size); diff --git a/include/fluent-bit/flb_mp_chunk.h b/include/fluent-bit/flb_mp_chunk.h index 7f3ff751141..86e59e075c6 100644 --- a/include/fluent-bit/flb_mp_chunk.h +++ b/include/fluent-bit/flb_mp_chunk.h @@ -30,6 +30,7 @@ /* Forward declaration to avoid circular dependencies */ struct flb_condition; +struct flb_mp_record_block; struct flb_mp_chunk_record { int modified; @@ -41,6 +42,9 @@ struct flb_mp_chunk_record { int owns_group_metadata; int owns_group_attributes; struct cfl_list _head; + + /* Pool block holding this record, NULL when allocated individually. */ + struct flb_mp_record_block *storage_block; }; struct flb_mp_chunk_cobj { @@ -56,6 +60,15 @@ struct flb_mp_chunk_cobj { /* Condition for filtering records during processing */ struct flb_condition *condition; + + /* Opt-in, bounded native value storage for the final serialization stage. */ + struct cfl_arena *value_pool; + size_t value_pool_bytes; + + /* Reference-counted wrapper blocks remain valid when records move. */ + int record_pool_enabled; + size_t record_pool_blocks; + struct flb_mp_record_block *record_pool; }; @@ -70,6 +83,12 @@ struct flb_mp_chunk_cobj *flb_mp_chunk_cobj_create(struct flb_log_event_encoder struct flb_log_event_decoder *log_decoder); int flb_mp_chunk_cobj_destroy(struct flb_mp_chunk_cobj *chunk_cobj); +/* Only enable when CFL values will be read and serialized, without mutation by + * a later native processor. The chunk owns the arena until destruction. */ +void flb_mp_chunk_cobj_value_pool_enable(struct flb_mp_chunk_cobj *chunk_cobj); +struct cfl_arena *flb_mp_chunk_cobj_value_pool_get(struct flb_mp_chunk_cobj *chunk_cobj, + size_t estimated_bytes); + int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size); int flb_mp_chunk_cobj_count_log_records(struct flb_mp_chunk_cobj *chunk_cobj); int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj); diff --git a/src/flb_mp.c b/src/flb_mp.c index d17f8c04c30..caa33018b9a 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -46,14 +47,204 @@ int flb_mp_count(const void *data, size_t bytes) return flb_mp_count_remaining(data, bytes, NULL); } +/* + * Validate a MessagePack value with a read-only traversal: no msgpack object + * tree or zone is constructed. Unusual or deeper inputs are rejected so the + * counting API can fall back to the event decoder. + */ +static int scan_log_value(mpack_reader_t *reader, mpack_tag_t tag, int depth, int typed) +{ + int ret; + uint32_t index; + uint32_t count; + mpack_tag_t key; + + if (depth >= 64 || mpack_reader_error(reader) != mpack_ok) { + return -1; + } + + switch (mpack_tag_type(&tag)) { + case mpack_type_array: + count = mpack_tag_array_count(&tag); + for (index = 0; index < count; index++) { + ret = scan_log_value(reader, mpack_read_tag(reader), depth + 1, typed); + if (ret != 0) { + return -1; + } + } + mpack_done_array(reader); + break; + case mpack_type_map: + count = mpack_tag_map_count(&tag); + for (index = 0; index < count; index++) { + key = mpack_read_tag(reader); + + /* typed mode requires string keys */ + if (typed && mpack_tag_type(&key) != mpack_type_str) { + return -1; + } + + if (scan_log_value(reader, key, depth + 1, 0) != 0) { + return -1; + } + + ret = scan_log_value(reader, mpack_read_tag(reader), depth + 1, typed); + if (ret != 0) { + return -1; + } + } + mpack_done_map(reader); + break; + case mpack_type_str: + mpack_skip_bytes(reader, mpack_tag_str_length(&tag)); + mpack_done_str(reader); + break; + case mpack_type_bin: + mpack_skip_bytes(reader, mpack_tag_bin_length(&tag)); + mpack_done_bin(reader); + break; + case mpack_type_ext: + if (typed) { + return -1; + } + mpack_skip_bytes(reader, mpack_tag_ext_length(&tag)); + mpack_done_ext(reader); + break; + default: + break; + } + + if (mpack_reader_error(reader) != mpack_ok) { + return -1; + } + + return 0; +} + +int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *records, int *plain) +{ + mpack_reader_t reader; + mpack_tag_t tag; + msgpack_object timestamp = {0}; + struct flb_time time; + char extension[8]; + int header; + int count = 0; + int ungrouped = FLB_TRUE; + int ret = -1; + + if (!data && bytes) { + return -1; + } + + mpack_reader_init_data(&reader, data, bytes); + + while (mpack_reader_remaining(&reader, NULL) > 0) { + /* every event is a two entries array: [header|timestamp, body] */ + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_array || mpack_tag_array_count(&tag) != 2) { + goto done; + } + + /* the header form is [timestamp, metadata] */ + tag = mpack_read_tag(&reader); + header = mpack_tag_type(&tag) == mpack_type_array; + if (header) { + if (mpack_tag_array_count(&tag) != 2) { + goto done; + } + tag = mpack_read_tag(&reader); + } + + if (mpack_tag_type(&tag) == mpack_type_uint) { + timestamp.type = MSGPACK_OBJECT_POSITIVE_INTEGER; + timestamp.via.u64 = mpack_tag_uint_value(&tag); + } + else if (mpack_tag_type(&tag) == mpack_type_ext && + mpack_tag_ext_length(&tag) == 8 && mpack_tag_ext_exttype(&tag) == 0) { + mpack_read_bytes(&reader, extension, sizeof(extension)); + mpack_done_ext(&reader); + + timestamp.type = MSGPACK_OBJECT_EXT; + timestamp.via.ext.ptr = extension; + timestamp.via.ext.size = sizeof(extension); + timestamp.via.ext.type = 0; + } + else { + /* the legacy decoder remains authoritative for float/int timestamps */ + goto done; + } + + if (mpack_reader_error(&reader) != mpack_ok || + flb_log_event_decoder_decode_timestamp(×tamp, &time) != 0) { + goto done; + } + + /* metadata */ + if (header) { + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_map || + scan_log_value(&reader, tag, 0, typed) != 0) { + goto done; + } + mpack_done_array(&reader); + } + + /* body */ + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_map || + scan_log_value(&reader, tag, 0, typed) != 0) { + goto done; + } + mpack_done_array(&reader); + + if (mpack_reader_error(&reader) != mpack_ok) { + goto done; + } + + /* group markers are not counted as records */ + if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_START || + time.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { + ungrouped = FLB_FALSE; + } + else if (flb_time_is_valid_eventtime(&time) == FLB_TRUE && count < INT_MAX) { + count++; + } + else { + goto done; + } + } + + *records = count; + *plain = ungrouped; + ret = 0; + +done: + /* + * Flag an error on early exits so the reader does not complain about + * unfinished compound types when it is destroyed. + */ + if (ret != 0) { + mpack_reader_flag_error(&reader, mpack_error_invalid); + } + mpack_reader_destroy(&reader); + + return ret; +} + /* Return the number of log events excluding group markers */ int flb_mp_count_log_records(const void *data, size_t bytes) { int count; int ret; + int plain; struct flb_log_event log_event; struct flb_log_event_decoder decoder; + if (flb_mp_scan_log_records(data, bytes, FLB_FALSE, &count, &plain) == 0) { + return count; + } + count = 0; ret = flb_log_event_decoder_init(&decoder, (char *) data, bytes); @@ -1030,210 +1221,288 @@ void flb_mp_accessor_destroy(struct flb_mp_accessor *mpa) } -static int mp_object_to_cfl(void **ptr, msgpack_object *o) +/* + * Convert a msgpack object into a CFL variant. When 'arena' is set, the + * variant and its children are allocated from it, otherwise from the heap. + */ +static struct cfl_variant *mp_variant_to_cfl(msgpack_object *object, struct cfl_arena *arena) { - int i; - int ret = -1; + struct cfl_variant *value; + struct cfl_variant *child; struct cfl_array *array; - struct cfl_kvlist *kvlist; - void *var; - msgpack_object key; - msgpack_object val; + struct cfl_kvlist *map; + msgpack_object_kv *pair; + size_t index; - switch(o->type) { + switch (object->type) { case MSGPACK_OBJECT_NIL: - var = cfl_variant_create_from_null(); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_null_in(arena); case MSGPACK_OBJECT_BOOLEAN: - var = cfl_variant_create_from_bool(o->via.boolean); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_bool_in(arena, object->via.boolean); case MSGPACK_OBJECT_POSITIVE_INTEGER: - var = cfl_variant_create_from_uint64(o->via.u64); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_uint64_in(arena, object->via.u64); case MSGPACK_OBJECT_NEGATIVE_INTEGER: - var = cfl_variant_create_from_int64(o->via.i64); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_int64_in(arena, object->via.i64); case MSGPACK_OBJECT_FLOAT32: case MSGPACK_OBJECT_FLOAT64: - var = cfl_variant_create_from_double(o->via.f64); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_double_in(arena, object->via.f64); case MSGPACK_OBJECT_STR: - var = cfl_variant_create_from_string_s((char *) o->via.str.ptr, - o->via.str.size, CFL_TRUE); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_string_s_in(arena, (char *) object->via.str.ptr, + object->via.str.size, CFL_TRUE); case MSGPACK_OBJECT_BIN: - var = cfl_variant_create_from_bytes((char *) o->via.str.ptr, - o->via.str.size, CFL_TRUE); - if (!var) { - return -1; - } - - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; - case MSGPACK_OBJECT_EXT: - /* we do not pack extension type content */ - *ptr = NULL; - ret = CFL_OBJECT_NONE; - break; + return cfl_variant_create_from_bytes_in(arena, (char *) object->via.bin.ptr, + object->via.bin.size, CFL_TRUE); case MSGPACK_OBJECT_ARRAY: - array = cfl_array_create(o->via.map.size); + array = cfl_array_create_in(arena, object->via.array.size); if (!array) { - return -1; + return NULL; } - ret = 0; - for (i = 0; i < o->via.map.size; i++) { - ret = mp_object_to_cfl((void *) &var, &o->via.array.ptr[i]); - if (ret == CFL_OBJECT_KVLIST) { - ret = cfl_array_append_kvlist(array, var); - } - else if (ret == CFL_OBJECT_VARIANT) { - ret = cfl_array_append(array, var); - } - else if (ret == CFL_OBJECT_ARRAY) { - ret = cfl_array_append_array(array, var); - } - else { - ret = -1; - break; + for (index = 0; index < object->via.array.size; index++) { + child = mp_variant_to_cfl(&object->via.array.ptr[index], arena); + if (!child || cfl_array_append(array, child) != 0) { + cfl_variant_destroy(child); + cfl_array_destroy(array); + return NULL; } } - if (ret == -1) { + value = cfl_variant_create_from_array_in(arena, array); + if (!value) { cfl_array_destroy(array); - return -1; } - - *ptr = array; - ret = CFL_OBJECT_ARRAY; - break; + return value; case MSGPACK_OBJECT_MAP: - kvlist = cfl_kvlist_create(); - if (!kvlist) { - return -1; + map = cfl_kvlist_create_in(arena); + if (!map) { + return NULL; } - ret = 0; - for (i = 0; i < o->via.map.size; i++) { - key = o->via.map.ptr[i].key; - val = o->via.map.ptr[i].val; - - /* force key type to be strin, otherwise just abort */ - if (key.type != MSGPACK_OBJECT_STR) { - ret = -1; - break; - } + for (index = 0; index < object->via.map.size; index++) { + pair = &object->via.map.ptr[index]; - /* key variant is ready, now we need the value variant */ - ret = mp_object_to_cfl((void *) &var, &val); - if (ret == -1) { - break; + /* only string keys are supported */ + if (pair->key.type != MSGPACK_OBJECT_STR) { + cfl_kvlist_destroy(map); + return NULL; } - if (ret == CFL_OBJECT_KVLIST) { - ret = cfl_kvlist_insert_kvlist_s(kvlist, - (char *) key.via.str.ptr, key.via.str.size, - var); - } - else if (ret == CFL_OBJECT_VARIANT) { - ret = cfl_kvlist_insert_s(kvlist, - (char *) key.via.str.ptr, key.via.str.size, - var); - } - else if (ret == CFL_OBJECT_ARRAY) { - ret = cfl_kvlist_insert_array_s(kvlist, - (char *) key.via.str.ptr, key.via.str.size, - var); - } - else { - ret = -1; - break; + child = mp_variant_to_cfl(&pair->val, arena); + if (!child || cfl_kvlist_insert_s(map, (char *) pair->key.via.str.ptr, + pair->key.via.str.size, child) != 0) { + cfl_variant_destroy(child); + cfl_kvlist_destroy(map); + return NULL; } } - if (ret == -1) { - cfl_kvlist_destroy(kvlist); - return -1; + value = cfl_variant_create_from_kvlist_in(arena, map); + if (!value) { + cfl_kvlist_destroy(map); } - - *ptr = kvlist; - ret = CFL_OBJECT_KVLIST; - break; + return value; default: - break; + /* extension types are not converted */ + return NULL; } - - return ret; } +/* Only maps (kvlist) and arrays can be converted to a cfl_object */ +static struct cfl_object *mp_object_to_cfl_in(msgpack_object *object, struct cfl_arena *arena) +{ + struct cfl_object *result; + + if (object->type != MSGPACK_OBJECT_MAP && object->type != MSGPACK_OBJECT_ARRAY) { + return NULL; + } + + result = cfl_object_create(); + if (!result) { + return NULL; + } + + result->variant = mp_variant_to_cfl(object, arena); + if (!result->variant) { + cfl_object_destroy(result); + return NULL; + } + + if (object->type == MSGPACK_OBJECT_MAP) { + result->type = CFL_OBJECT_KVLIST; + } + else { + result->type = CFL_OBJECT_ARRAY; + } + + return result; +} /* Convert a msgpack object to a cfl_object */ struct cfl_object *flb_mp_object_to_cfl(msgpack_object *o) { - int ret; - void *out = NULL; - struct cfl_object *obj; + return mp_object_to_cfl_in(o, NULL); +} + +/* + * Bytes a chunk value pool can hold before new record trees use the heap, and + * its arena block size. Trees already in the pool can keep growing, so edits + * on them never fail because the pool is full. + */ +#define FLB_MP_VALUE_POOL_LIMIT (1024 * 1024) +#define FLB_MP_VALUE_POOL_BLOCK 16384 - /* For now, only allow to convert to map (kvlist) or array */ - if (o->type != MSGPACK_OBJECT_MAP && o->type != MSGPACK_OBJECT_ARRAY) { +/* Prefix of every pool allocation, it keeps the size for accounting */ +union value_pool_header { + size_t size; + long double alignment; + void *pointer; +}; + +static void *value_pool_malloc(void *context, size_t size) +{ + struct flb_mp_chunk_cobj *chunk = context; + union value_pool_header *header; + + if (size > SIZE_MAX - sizeof(*header)) { return NULL; } - obj = cfl_object_create(); - if (!obj) { + header = flb_malloc(sizeof(*header) + size); + if (!header) { + flb_errno(); return NULL; } - ret = mp_object_to_cfl(&out, o); - if (ret < 0) { - cfl_object_destroy(obj); + header->size = sizeof(*header) + size; + chunk->value_pool_bytes += header->size; + + return header + 1; +} + +static void value_pool_free(void *context, void *pointer) +{ + struct flb_mp_chunk_cobj *chunk = context; + union value_pool_header *header = (union value_pool_header *) pointer - 1; + + chunk->value_pool_bytes -= header->size; + flb_free(header); +} + +void flb_mp_chunk_cobj_value_pool_enable(struct flb_mp_chunk_cobj *chunk) +{ + struct cfl_arena_options options; + + if (chunk->value_pool) { + return; + } + + /* fixed size blocks, accounted against the chunk through the hooks */ + cfl_arena_options_init(&options); + options.chunk_size = FLB_MP_VALUE_POOL_BLOCK; + options.maximum_chunk_size = FLB_MP_VALUE_POOL_BLOCK; + options.large_object_threshold = FLB_MP_VALUE_POOL_BLOCK; + options.malloc_fn = value_pool_malloc; + options.free_fn = value_pool_free; + options.allocator_context = chunk; + + chunk->value_pool = cfl_arena_create_with_options(&options); + if (chunk->value_pool) { + cfl_arena_external_cache_limit_set(chunk->value_pool, 0); + } +} + +struct cfl_arena *flb_mp_chunk_cobj_value_pool_get(struct flb_mp_chunk_cobj *chunk, + size_t estimated_bytes) +{ + size_t available; + + if (!chunk || !chunk->value_pool || + chunk->value_pool_bytes >= FLB_MP_VALUE_POOL_LIMIT) { return NULL; } - ret = cfl_object_set(obj, ret, out); - if (ret == -1) { - if (ret == CFL_OBJECT_KVLIST) { - cfl_kvlist_destroy(out); + available = FLB_MP_VALUE_POOL_LIMIT - chunk->value_pool_bytes; + + /* + * Leave room for a new arena block and its allocation headers. Oversized + * roots use heap ownership as a whole, never mixed container ownership. + */ + if (available < 2 * FLB_MP_VALUE_POOL_BLOCK || + estimated_bytes > available - 2 * FLB_MP_VALUE_POOL_BLOCK) { + return NULL; + } + + return chunk->value_pool; +} + +/* + * Upper estimate for arena nodes, aligned slots, kvpairs and copied map keys. + * MessagePack string/byte payloads themselves remain borrowed from the decoder. + */ +static size_t mp_value_pool_cost(msgpack_object *object, int depth) +{ + size_t cost = 512; + size_t child; + size_t index; + msgpack_object *value; + + if (depth >= 64) { + return SIZE_MAX; + } + + if (object->type != MSGPACK_OBJECT_MAP && object->type != MSGPACK_OBJECT_ARRAY) { + return cost; + } + + /* maps and arrays share the same 'size' layout in the msgpack union */ + for (index = 0; index < object->via.array.size; index++) { + if (object->type == MSGPACK_OBJECT_MAP) { + value = &object->via.map.ptr[index].val; + + child = 0; + if (object->via.map.ptr[index].key.type == MSGPACK_OBJECT_STR) { + child = object->via.map.ptr[index].key.via.str.size; + } + + if (cost > SIZE_MAX - 512 || child > (SIZE_MAX - cost - 512) / 2) { + return SIZE_MAX; + } + cost += 512 + 2 * child; } - else if (ret == CFL_OBJECT_ARRAY) { - cfl_array_destroy(out); + else { + value = &object->via.array.ptr[index]; } - cfl_object_destroy(obj); - return NULL; + + child = mp_value_pool_cost(value, depth + 1); + if (child > SIZE_MAX - cost) { + return SIZE_MAX; + } + cost += child; } - return obj; + return cost; +} + +/* + * Convert a decoded msgpack object for a chunk record. The chunk value pool is + * used when enabled and large enough, falling back to heap allocations. + */ +static struct cfl_object *chunk_object_from_msgpack(struct flb_mp_chunk_cobj *chunk, + msgpack_object *object) +{ + struct cfl_arena *arena = NULL; + struct cfl_object *result; + + if (chunk->value_pool) { + arena = flb_mp_chunk_cobj_value_pool_get(chunk, mp_value_pool_cost(object, 0)); + } + + result = mp_object_to_cfl_in(object, arena); + if (!result && arena) { + result = mp_object_to_cfl_in(object, NULL); + } + + return result; } static int mp_cfl_to_msgpack(struct cfl_variant *var, @@ -1347,20 +1616,76 @@ int flb_mp_cfl_to_msgpack(struct cfl_object *obj, char **out_buf, size_t *out_si return 0; } +/* Records per pool block and maximum number of blocks per chunk */ +#define FLB_MP_RECORD_BLOCK_ENTRIES 64 +#define FLB_MP_RECORD_BLOCK_LIMIT 8 + +/* + * A block has one owner reference plus one reference per live record. Moving + * records between chunks does not change their storage lifetime. + */ +struct flb_mp_record_block { + struct flb_mp_record_block *next; + size_t references; + size_t used; + struct flb_mp_chunk_record records[FLB_MP_RECORD_BLOCK_ENTRIES]; +}; + +static void record_block_release(struct flb_mp_record_block *block) +{ + block->references--; + if (block->references == 0) { + flb_free(block); + } +} + +static void record_storage_release(struct flb_mp_chunk_record *record) +{ + if (record->storage_block) { + record_block_release(record->storage_block); + } + else { + flb_free(record); + } +} + struct flb_mp_chunk_record *flb_mp_chunk_record_create(struct flb_mp_chunk_cobj *chunk_cobj) { struct flb_mp_chunk_record *record; + struct flb_mp_record_block *block; + + if (chunk_cobj && chunk_cobj->record_pool_enabled) { + block = chunk_cobj->record_pool; + + /* the current block is full (or missing), try to add a new one */ + if ((!block || block->used == FLB_MP_RECORD_BLOCK_ENTRIES) && + chunk_cobj->record_pool_blocks < FLB_MP_RECORD_BLOCK_LIMIT) { + block = flb_calloc(1, sizeof(struct flb_mp_record_block)); + if (!block) { + flb_errno(); + } + else { + block->references = 1; + block->next = chunk_cobj->record_pool; + chunk_cobj->record_pool = block; + chunk_cobj->record_pool_blocks++; + } + } + + if (block && block->used < FLB_MP_RECORD_BLOCK_ENTRIES) { + record = &block->records[block->used++]; + record->storage_block = block; + block->references++; + return record; + } + } + /* pool disabled or exhausted: allocate the record individually */ record = flb_calloc(1, sizeof(struct flb_mp_chunk_record)); if (!record) { flb_errno(); return NULL; } - record->modified = FLB_FALSE; - record->cobj_group_metadata = NULL; - record->cobj_group_attributes = NULL; - record->owns_group_metadata = FLB_FALSE; - record->owns_group_attributes = FLB_FALSE; return record; } @@ -1389,124 +1714,106 @@ struct flb_mp_chunk_cobj *flb_mp_chunk_cobj_create(struct flb_log_event_encoder return chunk_cobj; } -static int generate_empty_msgpack_map(char **out_buf, size_t *out_size) +/* + * Pack a record field into the scratch buffer, an empty map when the field is + * not set. The buffer capacity is reused by every field of an encode call. + */ +static int encode_cfl_field(struct cfl_object *obj, msgpack_sbuffer *buffer, + msgpack_packer *packer) { - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; + msgpack_sbuffer_clear(buffer); - /* initialize msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 0); + if (!obj) { + return msgpack_pack_map(packer, 0); + } - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; + if (obj->type == CFL_OBJECT_NONE || !obj->variant) { + return -1; + } - return 0; + return mp_cfl_to_msgpack(obj->variant, buffer, packer); } int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size) { int ret; - int record_type; - char *mp_buf; - size_t mp_size; struct cfl_list *head; struct flb_mp_chunk_record *record; + struct cfl_object *body; + msgpack_sbuffer buffer; + msgpack_packer packer; if (!chunk_cobj) { return -1; } + msgpack_sbuffer_init(&buffer); + msgpack_packer_init(&packer, &buffer, msgpack_sbuffer_write); + /* Iterate all records */ cfl_list_foreach(head, &chunk_cobj->records) { record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); ret = flb_log_event_encoder_begin_record(chunk_cobj->log_encoder); - if (ret == -1) { - return -1; + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto failure; } - ret = flb_log_event_encoder_set_timestamp(chunk_cobj->log_encoder, &record->event.timestamp); - if (ret == -1) { - return -1; + ret = flb_log_event_encoder_set_timestamp(chunk_cobj->log_encoder, + &record->event.timestamp); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto failure; } - /* Determine record type from timestamp */ - if (record->event.timestamp.tm.tv_sec >= 0) { - record_type = FLB_LOG_EVENT_NORMAL; - } - else if (record->event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_START) { - record_type = FLB_LOG_EVENT_GROUP_START; - } - else if (record->event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { - record_type = FLB_LOG_EVENT_GROUP_END; + /* metadata */ + if (encode_cfl_field(record->cobj_metadata, &buffer, &packer) != 0) { + goto failure; } - else { - record_type = FLB_LOG_EVENT_NORMAL; - } - - if (record->cobj_metadata) { - ret = flb_mp_cfl_to_msgpack(record->cobj_metadata, &mp_buf, &mp_size); - if (ret == -1) { - return -1; - } - } - else { - ret = generate_empty_msgpack_map(&mp_buf, &mp_size); - if (ret == -1) { - return -1; - } - } - - ret = flb_log_event_encoder_set_metadata_from_raw_msgpack(chunk_cobj->log_encoder, mp_buf, mp_size); + ret = flb_log_event_encoder_set_metadata_from_raw_msgpack(chunk_cobj->log_encoder, + buffer.data, buffer.size); if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_free(mp_buf); - return -1; + goto failure; } - flb_free(mp_buf); - /* For group start records, use group attributes as body if available */ - if (record_type == FLB_LOG_EVENT_GROUP_START && record->cobj_group_attributes) { - ret = flb_mp_cfl_to_msgpack(record->cobj_group_attributes, &mp_buf, &mp_size); - if (ret == -1) { - return -1; - } + /* for group start records, use group attributes as body if available */ + body = record->cobj_record; + if (record->event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_START && + record->cobj_group_attributes) { + body = record->cobj_group_attributes; } - else if (record->cobj_record) { - ret = flb_mp_cfl_to_msgpack(record->cobj_record, &mp_buf, &mp_size); - if (ret == -1) { - return -1; - } - } - else { - ret = generate_empty_msgpack_map(&mp_buf, &mp_size); - if (ret == -1) { - return -1; - } + + if (encode_cfl_field(body, &buffer, &packer) != 0) { + goto failure; } - ret = flb_log_event_encoder_set_body_from_raw_msgpack(chunk_cobj->log_encoder, mp_buf, mp_size); + ret = flb_log_event_encoder_set_body_from_raw_msgpack(chunk_cobj->log_encoder, + buffer.data, buffer.size); if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_free(mp_buf); - return -1; + goto failure; } - flb_free(mp_buf); ret = flb_log_event_encoder_commit_record(chunk_cobj->log_encoder); - if (ret == -1) { - return -1; + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto failure; } } + msgpack_sbuffer_destroy(&buffer); + /* set new output buffer */ *out_buf = chunk_cobj->log_encoder->output_buffer; *out_size = chunk_cobj->log_encoder->output_length; flb_log_event_encoder_claim_internal_buffer_ownership(chunk_cobj->log_encoder); return 0; + +failure: + /* never leave partial records behind for the next chunk using this encoder */ + flb_log_event_encoder_reset(chunk_cobj->log_encoder); + msgpack_sbuffer_destroy(&buffer); + + return -1; } int flb_mp_chunk_cobj_count_log_records(struct flb_mp_chunk_cobj *chunk_cobj) @@ -1677,6 +1984,7 @@ int flb_mp_chunk_cobj_destroy(struct flb_mp_chunk_cobj *chunk_cobj) struct cfl_list *tmp; struct cfl_list *head; struct flb_mp_chunk_record *record; + struct flb_mp_record_block *block; if (!chunk_cobj) { return -1; @@ -1699,9 +2007,17 @@ int flb_mp_chunk_cobj_destroy(struct flb_mp_chunk_cobj *chunk_cobj) cfl_object_destroy(record->cobj_group_attributes); } cfl_list_del(&record->_head); - flb_free(record); + record_storage_release(record); + } + + /* drop the owner reference, blocks still used by moved records survive */ + while (chunk_cobj->record_pool) { + block = chunk_cobj->record_pool; + chunk_cobj->record_pool = block->next; + record_block_release(block); } + cfl_arena_destroy(chunk_cobj->value_pool); flb_free(chunk_cobj); return 0; } @@ -1747,20 +2063,21 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, ret = flb_log_event_decoder_next(chunk_cobj->log_decoder, &record->event); if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_free(record); + record_storage_release(record); return -1; } - record->cobj_metadata = flb_mp_object_to_cfl(record->event.metadata); + record->cobj_metadata = chunk_object_from_msgpack(chunk_cobj, + record->event.metadata); if (!record->cobj_metadata) { - flb_free(record); + record_storage_release(record); return FLB_MP_CHUNK_RECORD_ERROR; } - record->cobj_record = flb_mp_object_to_cfl(record->event.body); + record->cobj_record = chunk_object_from_msgpack(chunk_cobj, record->event.body); if (!record->cobj_record) { cfl_object_destroy(record->cobj_metadata); - flb_free(record); + record_storage_release(record); return -1; } @@ -1768,7 +2085,7 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, if (ret != FLB_EVENT_DECODER_SUCCESS) { cfl_object_destroy(record->cobj_record); cfl_object_destroy(record->cobj_metadata); - flb_free(record); + record_storage_release(record); return FLB_MP_CHUNK_RECORD_ERROR; } @@ -1804,14 +2121,15 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, record->event.group_metadata && (record->event.group_metadata->type == MSGPACK_OBJECT_MAP || record->event.group_metadata->type == MSGPACK_OBJECT_ARRAY)) { - record->cobj_group_metadata = flb_mp_object_to_cfl(record->event.group_metadata); + record->cobj_group_metadata = + chunk_object_from_msgpack(chunk_cobj, record->event.group_metadata); if (!record->cobj_group_metadata) { if (record->owns_group_attributes && record->cobj_group_attributes) { cfl_object_destroy(record->cobj_group_attributes); } cfl_object_destroy(record->cobj_record); cfl_object_destroy(record->cobj_metadata); - flb_free(record); + record_storage_release(record); return FLB_MP_CHUNK_RECORD_ERROR; } record->owns_group_metadata = FLB_TRUE; @@ -1824,14 +2142,15 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, record->event.group_attributes && (record->event.group_attributes->type == MSGPACK_OBJECT_MAP || record->event.group_attributes->type == MSGPACK_OBJECT_ARRAY)) { - record->cobj_group_attributes = flb_mp_object_to_cfl(record->event.group_attributes); + record->cobj_group_attributes = + chunk_object_from_msgpack(chunk_cobj, record->event.group_attributes); if (!record->cobj_group_attributes) { if (record->owns_group_metadata && record->cobj_group_metadata) { cfl_object_destroy(record->cobj_group_metadata); } cfl_object_destroy(record->cobj_record); cfl_object_destroy(record->cobj_metadata); - flb_free(record); + record_storage_release(record); return FLB_MP_CHUNK_RECORD_ERROR; } record->owns_group_attributes = FLB_TRUE; @@ -1967,7 +2286,7 @@ int flb_mp_chunk_cobj_record_destroy(struct flb_mp_chunk_cobj *chunk_cobj, } cfl_list_del(&record->_head); - flb_free(record); + record_storage_release(record); return 0; } From a3465dd6f9b1f12dbbce5ea2979323770682275f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 04/61] log_event_decoder: reuse the unpack zone between records Signed-off-by: Eduardo Silva --- src/flb_log_event_decoder.c | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/src/flb_log_event_decoder.c b/src/flb_log_event_decoder.c index fa96eb07842..cfd8920fe00 100644 --- a/src/flb_log_event_decoder.c +++ b/src/flb_log_event_decoder.c @@ -370,10 +370,31 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, context->record_length = 0; previous_offset = context->offset; - result = msgpack_unpack_next(&context->unpacked_event, - context->buffer, - context->length, - &context->offset); + + /* + * Reuse one zone across records. Clearing keeps its first chunk, whereas + * msgpack_unpack_next() frees and recreates the zone for every record. A + * group start takes the zone over, so a new one is created on the next call. + */ + if (context->unpacked_event.zone == NULL) { + context->unpacked_event.zone = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if (context->unpacked_event.zone == NULL) { + context->last_result = FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE; + return context->last_result; + } + } + else { + msgpack_zone_clear(context->unpacked_event.zone); + } + + memset(&context->unpacked_event.data, 0, sizeof(msgpack_object)); + result = msgpack_unpack(context->buffer, context->length, &context->offset, + context->unpacked_event.zone, &context->unpacked_event.data); + + /* remaining bytes belong to the next records */ + if (result == MSGPACK_UNPACK_EXTRA_BYTES) { + result = MSGPACK_UNPACK_SUCCESS; + } if (result == MSGPACK_UNPACK_CONTINUE) { context->last_result = FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA; @@ -460,9 +481,8 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, if (context->read_groups != FLB_TRUE) { /* - * Skip group markers and move to the next record. - * msgpack_unpack_next will properly destroy and reinitialize - * unpacked_event, so no explicit cleanup needed here. + * Skip group markers and move to the next record. The next + * iteration clears or recreates the unpacked_event zone. */ memset(event, 0, sizeof(struct flb_log_event)); goto next_record; From 3d8bdbac539b8fb22a4b69af1ba1b930c47b91e5 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 05/61] json: render mutable values with a heap stack Output workers run on small coroutine stacks; avoid recursion when rendering nested values. Signed-off-by: Eduardo Silva --- src/flb_json.c | 159 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 109 insertions(+), 50 deletions(-) diff --git a/src/flb_json.c b/src/flb_json.c index 7bf452fe521..3e8e1b5cb65 100644 --- a/src/flb_json.c +++ b/src/flb_json.c @@ -511,62 +511,18 @@ static yyjson_mut_val *msgpack_to_yyjson_mut(yyjson_mut_doc *document, */ #define FLB_JSON_MUT_RENDER_MAX_DEPTH 1024 -static yyjson_mut_val *mutable_to_yyjson_mut_internal(yyjson_mut_doc *document, - struct flb_json_mut_val *value, - size_t depth) +static yyjson_mut_val *mutable_value_to_yyjson(yyjson_mut_doc *document, + struct flb_json_mut_val *value) { - struct flb_json_mut_kv *kv_entry; - struct flb_json_mut_entry *array_entry; - yyjson_mut_val *result; - yyjson_mut_val *item; - - if (depth > FLB_JSON_MUT_RENDER_MAX_DEPTH) { + if (value == NULL) { return NULL; } switch (value->type) { case FLB_JSON_MUT_OBJECT: - result = yyjson_mut_obj(document); - if (result == NULL) { - return NULL; - } - - for (kv_entry = value->data.object.head; - kv_entry != NULL; - kv_entry = kv_entry->next) { - item = mutable_to_yyjson_mut_internal(document, - kv_entry->value, - depth + 1); - if (item == NULL) { - return NULL; - } - - if (!yyjson_mut_obj_add(result, - yyjson_mut_strcpy(document, kv_entry->key), - item)) { - return NULL; - } - } - - return result; + return yyjson_mut_obj(document); case FLB_JSON_MUT_ARRAY: - result = yyjson_mut_arr(document); - if (result == NULL) { - return NULL; - } - - for (array_entry = value->data.array.head; - array_entry != NULL; - array_entry = array_entry->next) { - item = mutable_to_yyjson_mut_internal(document, - array_entry->value, - depth + 1); - if (item == NULL || !yyjson_mut_arr_add_val(result, item)) { - return NULL; - } - } - - return result; + return yyjson_mut_arr(document); case FLB_JSON_MUT_STRING: return yyjson_mut_strcpy(document, value->data.string); case FLB_JSON_MUT_BOOL: @@ -584,10 +540,113 @@ static yyjson_mut_val *mutable_to_yyjson_mut_internal(yyjson_mut_doc *document, } } +/* + * Output workers run on small coroutine stacks, so the traversal keeps its own + * stack on the heap: even modest OTLP nesting expands into many JSON object and + * array layers. + */ static yyjson_mut_val *mutable_to_yyjson_mut(yyjson_mut_doc *document, struct flb_json_mut_val *value) { - return mutable_to_yyjson_mut_internal(document, value, 0); + struct render_frame { + yyjson_mut_val *output; + struct flb_json_mut_kv *object; + struct flb_json_mut_entry *array; + }; + int ret; + struct render_frame *frames; + struct render_frame *frame; + struct flb_json_mut_val *child; + yyjson_mut_val *result; + yyjson_mut_val *item; + const char *key; + size_t depth; + + result = mutable_value_to_yyjson(document, value); + if (result == NULL) { + return NULL; + } + + frames = flb_calloc(FLB_JSON_MUT_RENDER_MAX_DEPTH + 1, sizeof(*frames)); + if (frames == NULL) { + flb_errno(); + return NULL; + } + + /* the root frame iterates the children of the top level container */ + depth = 0; + frames[0].output = result; + if (value->type == FLB_JSON_MUT_OBJECT) { + frames[0].object = value->data.object.head; + } + else if (value->type == FLB_JSON_MUT_ARRAY) { + frames[0].array = value->data.array.head; + } + + while (1) { + frame = &frames[depth]; + key = NULL; + + /* take the next child of the current container, or pop the frame */ + if (frame->object != NULL) { + key = frame->object->key; + child = frame->object->value; + frame->object = frame->object->next; + } + else if (frame->array != NULL) { + child = frame->array->value; + frame->array = frame->array->next; + } + else { + if (depth == 0) { + break; + } + depth--; + continue; + } + + item = mutable_value_to_yyjson(document, child); + if (item == NULL) { + result = NULL; + break; + } + + if (key) { + ret = yyjson_mut_obj_add(frame->output, yyjson_mut_strcpy(document, key), item); + } + else { + ret = yyjson_mut_arr_add_val(frame->output, item); + } + + if (!ret) { + result = NULL; + break; + } + + /* containers get their own frame so their children are visited next */ + if (child->type == FLB_JSON_MUT_OBJECT || child->type == FLB_JSON_MUT_ARRAY) { + if (depth == FLB_JSON_MUT_RENDER_MAX_DEPTH) { + result = NULL; + break; + } + + frame = &frames[++depth]; + frame->output = item; + frame->object = NULL; + frame->array = NULL; + + if (child->type == FLB_JSON_MUT_OBJECT) { + frame->object = child->data.object.head; + } + else { + frame->array = child->data.array.head; + } + } + } + + flb_free(frames); + + return result; } static char *render_msgpack_document_yyjson(struct flb_json_doc *document, From b5aa52eafab2d34a8ea25b23ac5329e8f0b110fb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 06/61] processor: carry record counts and reset encoder on failure Carry known logical record counts through the processor chain so they are not recounted after every unit. On a failed chunk encode, reset the encoder and release the chunk so records cannot leak into the next chunk. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_processor.h | 6 +++ src/flb_processor.c | 84 +++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 8 deletions(-) diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index 2a56531cbf5..2a3805a9ab0 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -240,6 +240,12 @@ int flb_processor_run(struct flb_processor *proc, void **out_buf, size_t *out_size); +/* Carry known logical log counts through the pipeline. -1 means unknown. */ +int flb_processor_run_with_count(struct flb_processor *proc, size_t starting_stage, int type, + const char *tag, size_t tag_len, void *data, size_t data_size, + void **out_buf, size_t *out_size, + int input_records, int *output_records); + struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, int event_type, char *unit_name); diff --git a/src/flb_processor.c b/src/flb_processor.c index 7f04146ff6f..c5f6badcbdf 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -1320,6 +1320,18 @@ int flb_processor_run(struct flb_processor *proc, const char *tag, size_t tag_len, void *data, size_t data_size, void **out_buf, size_t *out_size) +{ + return flb_processor_run_with_count(proc, starting_stage, type, tag, tag_len, + data, data_size, out_buf, out_size, -1, NULL); +} + +int flb_processor_run_with_count(struct flb_processor *proc, + size_t starting_stage, + int type, + const char *tag, size_t tag_len, + void *data, size_t data_size, + void **out_buf, size_t *out_size, + int input_records, int *output_records) { int ret; int pu_error; @@ -1327,6 +1339,8 @@ int flb_processor_run(struct flb_processor *proc, int unit_out_items; int item_metrics_available; int finalize; + int log_buffer_normalized = FLB_FALSE; + int log_buffer_records = input_records; void *cur_buf = NULL; size_t cur_size; void *tmp_buf = NULL; @@ -1349,6 +1363,10 @@ int flb_processor_run(struct flb_processor *proc, char *name; #endif + if (output_records) { + *output_records = -1; + } + if (type == FLB_PROCESSOR_LOGS) { list = &proc->logs; } @@ -1395,7 +1413,12 @@ int flb_processor_run(struct flb_processor *proc, item_metrics_available = FLB_FALSE; if (type == FLB_PROCESSOR_LOGS) { - unit_in_items = flb_mp_count_log_records(cur_buf, cur_size); + if (!chunk_cobj && log_buffer_records >= 0) { + unit_in_items = log_buffer_records; + } + else { + unit_in_items = flb_mp_count_log_records(cur_buf, cur_size); + } unit_out_items = unit_in_items; item_metrics_available = FLB_TRUE; } @@ -1436,7 +1459,12 @@ int flb_processor_run(struct flb_processor *proc, } #ifdef FLB_HAVE_METRICS name = (char *) (flb_filter_name(f_ins)); - in_records = flb_mp_count_log_records(cur_buf, cur_size); + if (log_buffer_records >= 0) { + in_records = log_buffer_records; + } + else { + in_records = flb_mp_count_log_records(cur_buf, cur_size); + } cmt_counter_add(f_ins->cmt_records, ts, in_records, 1, (char *[]) {name}); cmt_counter_add(f_ins->cmt_bytes, ts, tmp_size, @@ -1500,6 +1528,8 @@ int flb_processor_run(struct flb_processor *proc, cur_buf = tmp_buf; cur_size = tmp_size; out_records = flb_mp_count_log_records(tmp_buf, tmp_size); + log_buffer_normalized = FLB_FALSE; + log_buffer_records = out_records; unit_out_items = out_records; #ifdef FLB_HAVE_METRICS if (out_records > in_records) { @@ -1610,9 +1640,19 @@ int flb_processor_run(struct flb_processor *proc, } if (finalize == FLB_TRUE) { - flb_mp_chunk_cobj_normalize_groups(chunk_cobj); + /* + * The encoder preserves normalized groups and the logical + * count. Reuse them until a filter replaces this serialized + * buffer. + */ + log_buffer_normalized = + flb_mp_chunk_cobj_normalize_groups(chunk_cobj) == 0; + log_buffer_records = flb_mp_chunk_cobj_count_log_records(chunk_cobj); if (cfl_list_size(&chunk_cobj->records) == 0) { + if (output_records) { + *output_records = 0; + } flb_log_event_encoder_reset(p_ins->log_encoder); flb_mp_chunk_cobj_destroy(chunk_cobj); @@ -1638,6 +1678,8 @@ int flb_processor_run(struct flb_processor *proc, ret = flb_mp_chunk_cobj_encode(chunk_cobj, (char **) &tmp_buf, &tmp_size); if (ret != 0) { flb_log_event_decoder_reset(p_ins->log_decoder, NULL, 0); + flb_log_event_encoder_reset(p_ins->log_encoder); + flb_mp_chunk_cobj_destroy(chunk_cobj); if (cur_buf != data) { flb_free(cur_buf); @@ -1677,7 +1719,12 @@ int flb_processor_run(struct flb_processor *proc, unit_out_items = flb_mp_chunk_cobj_count_log_records(chunk_cobj); } else if (cur_buf != NULL) { - unit_out_items = flb_mp_count_log_records(cur_buf, cur_size); + if (log_buffer_records >= 0) { + unit_out_items = log_buffer_records; + } + else { + unit_out_items = flb_mp_count_log_records(cur_buf, cur_size); + } } } } @@ -1704,7 +1751,16 @@ int flb_processor_run(struct flb_processor *proc, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); - out_buf = NULL; + if (cur_buf != data) { + cmt_destroy(cur_buf); + } + + if (out_buf != NULL) { + *out_buf = NULL; + } + if (out_size != NULL) { + *out_size = 0; + } return -1; } @@ -1810,7 +1866,7 @@ int flb_processor_run(struct flb_processor *proc, /* set output buffer */ if (type == FLB_PROCESSOR_LOGS && - cur_buf != NULL) { + cur_buf != NULL && !log_buffer_normalized) { ret = flb_mp_normalize_log_buffer_groups_msgpack(cur_buf, cur_size, (char **) &tmp_buf, &tmp_size); @@ -1821,11 +1877,15 @@ int flb_processor_run(struct flb_processor *proc, cur_buf = tmp_buf; cur_size = tmp_size; + log_buffer_records = -1; } } - if (type == FLB_PROCESSOR_LOGS && - (cur_buf == NULL || flb_mp_count_log_records(cur_buf, cur_size) == 0)) { + if (type == FLB_PROCESSOR_LOGS && cur_buf != NULL && log_buffer_records < 0) { + log_buffer_records = flb_mp_count_log_records(cur_buf, cur_size); + } + + if (type == FLB_PROCESSOR_LOGS && (cur_buf == NULL || log_buffer_records == 0)) { if (cur_buf != data) { flb_free(cur_buf); } @@ -1838,6 +1898,10 @@ int flb_processor_run(struct flb_processor *proc, *out_size = 0; } + if (output_records) { + *output_records = 0; + } + return 0; } @@ -1849,6 +1913,10 @@ int flb_processor_run(struct flb_processor *proc, *out_size = cur_size; } + if (output_records && type == FLB_PROCESSOR_LOGS) { + *output_records = log_buffer_records; + } + return 0; } From 56146e5c08b8d9c5afe443b238ab58481a3c8ec4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 07/61] output: use the serialized trace size for processed chunks Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_output.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index 6fce9a3bcd0..7f496e8a3c2 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -1151,6 +1151,8 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, return NULL; } + /* Capacity can exceed the serialized trace bytes after processing. */ + p_size = serialization_buffer_offset; out_flush->processed_event_chunk = flb_event_chunk_create( evc->type, 0, From 85d7fbee8407221db47d79fe82a59ac0f3b768dd Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 08/61] input_log: forward exact record counts to processors Signed-off-by: Eduardo Silva --- src/flb_input_log.c | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/flb_input_log.c b/src/flb_input_log.c index e14c84820c1..764c9893fc1 100644 --- a/src/flb_input_log.c +++ b/src/flb_input_log.c @@ -1526,13 +1526,20 @@ static int input_log_append_processed_internal(struct flb_input_instance *ins, return ret; } +/* + * 'exact_records' tells whether 'records' is the exact number of log records in + * the buffer. Caller-provided hints (for example, lines read by an input) are + * not forwarded to processors, which count the records themselves. + */ static int input_log_append(struct flb_input_instance *ins, size_t processor_starting_stage, - size_t records, + size_t records, int exact_records, const char *tag, size_t tag_len, const void *buf, size_t buf_size) { int ret; + int known_records = -1; + int processed_records = -1; int processor_is_active; void *out_buf = (void *) buf; size_t out_size = buf_size; @@ -1550,12 +1557,17 @@ static int input_log_append(struct flb_input_instance *ins, } } - ret = flb_processor_run(ins->processor, - processor_starting_stage, - FLB_PROCESSOR_LOGS, - tag, tag_len, - (char *) buf, buf_size, - &out_buf, &out_size); + if (exact_records && records <= INT_MAX) { + known_records = (int) records; + } + + ret = flb_processor_run_with_count(ins->processor, + processor_starting_stage, + FLB_PROCESSOR_LOGS, + tag, tag_len, + (char *) buf, buf_size, + &out_buf, &out_size, + known_records, &processed_records); if (ret == -1) { return -1; } @@ -1567,9 +1579,13 @@ static int input_log_append(struct flb_input_instance *ins, return 0; } - if (buf != out_buf) { - /* a new buffer was created, re-count the number of records */ - records = flb_mp_count_log_records(out_buf, out_size); + if (buf != out_buf || exact_records) { + if (processed_records >= 0) { + records = processed_records; + } + else if (buf != out_buf) { + records = flb_mp_count_log_records(out_buf, out_size); + } } } @@ -1604,7 +1620,7 @@ int flb_input_log_append(struct flb_input_instance *ins, size_t records; records = flb_mp_count_log_records(buf, buf_size); - ret = input_log_append(ins, 0, records, tag, tag_len, buf, buf_size); + ret = input_log_append(ins, 0, records, FLB_TRUE, tag, tag_len, buf, buf_size); return ret; } @@ -1636,6 +1652,7 @@ int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins, return input_log_append(ins, processor_starting_stage, flb_mp_count_log_records(buf, buf_size), + FLB_TRUE, tag, tag_len, buf, @@ -1650,6 +1667,6 @@ int flb_input_log_append_records(struct flb_input_instance *ins, { int ret; - ret = input_log_append(ins, 0, records, tag, tag_len, buf, buf_size); + ret = input_log_append(ins, 0, records, FLB_FALSE, tag, tag_len, buf, buf_size); return ret; } From 44ff4884cb8b4bb07674592f514c2874bc201c09 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 09/61] opentelemetry: preserve typed OTLP values Keep typed attributes and metadata for metrics, traces and logs. OTLP JSON metrics merge repeated series by labels as before, bytesValue is decoded as binary, and invalid attribute warnings are reported once per list. Signed-off-by: Eduardo Silva --- src/opentelemetry/flb_opentelemetry_logs.c | 17 +- src/opentelemetry/flb_opentelemetry_metrics.c | 329 ++++++++---------- .../flb_opentelemetry_otlp_json.c | 50 ++- src/opentelemetry/flb_opentelemetry_traces.c | 197 +---------- src/opentelemetry/flb_opentelemetry_utils.c | 235 +++++++++---- 5 files changed, 397 insertions(+), 431 deletions(-) diff --git a/src/opentelemetry/flb_opentelemetry_logs.c b/src/opentelemetry/flb_opentelemetry_logs.c index 63687cc833b..6253eb1db74 100644 --- a/src/opentelemetry/flb_opentelemetry_logs.c +++ b/src/opentelemetry/flb_opentelemetry_logs.c @@ -243,8 +243,11 @@ static int process_json_payload_log_records_entry( } } - /* traceFlags */ - result = flb_otel_utils_find_map_entry_by_key(log_records_entry, "traceFlags", 0, FLB_TRUE); + /* Canonical OTLP JSON flags, with legacy traceFlags compatibility. */ + result = flb_otel_utils_find_map_entry_by_key(log_records_entry, "flags", 0, FLB_TRUE); + if (result < 0) { + result = flb_otel_utils_find_map_entry_by_key(log_records_entry, "traceFlags", 0, FLB_TRUE); + } if (result >= 0) { trace_flags = &log_records_entry->ptr[result].val; } @@ -756,6 +759,16 @@ static int process_json_payload_resource_logs_entry (struct flb_log_event_encode } } + /* Preserve the scope's dropped attribute count like the resource's. */ + result = flb_otel_utils_find_map_entry_by_key(&scope->via.map, + "droppedAttributesCount", 0, FLB_TRUE); + if (result >= 0) { + obj = &scope->via.map.ptr[result].val; + flb_log_event_encoder_append_body_values(tmp_encoder, + FLB_LOG_EVENT_CSTRING_VALUE("dropped_attributes_count"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(obj)); + } + /* scope attributes */ result = flb_otel_utils_find_map_entry_by_key(&scope->via.map, "attributes", 0, FLB_TRUE); if (result >= 0) { diff --git a/src/opentelemetry/flb_opentelemetry_metrics.c b/src/opentelemetry/flb_opentelemetry_metrics.c index 61319eec9e2..43695aa455f 100644 --- a/src/opentelemetry/flb_opentelemetry_metrics.c +++ b/src/opentelemetry/flb_opentelemetry_metrics.c @@ -17,6 +17,7 @@ * limitations under the License. */ +#include #include #include #include @@ -25,6 +26,7 @@ #include #include +#include #include #include #include @@ -320,26 +322,6 @@ static int set_metric_unit(struct cmt_map *map, msgpack_object_map *metric_map) return 0; } -static char *map_type_to_key(int map_type) -{ - switch (map_type) { - case CMT_COUNTER: - return "counter"; - case CMT_GAUGE: - return "gauge"; - case CMT_UNTYPED: - return "untyped"; - case CMT_SUMMARY: - return "summary"; - case CMT_HISTOGRAM: - return "histogram"; - case CMT_EXP_HISTOGRAM: - return "exp_histogram"; - default: - return "unknown"; - } -} - static struct cfl_kvlist *get_or_create_external_metadata_kvlist( struct cfl_kvlist *root, char *key) { @@ -376,62 +358,6 @@ static struct cfl_kvlist *get_or_create_external_metadata_kvlist( * index of the entries created there during the request, keyed by parent * kvlist and lowercased key to keep the case-insensitive kvlist semantics. */ -static struct cfl_kvlist *get_or_create_indexed_kvlist(struct flb_hash_table *index, - struct cfl_kvlist *root, - char *key) -{ - int ret; - size_t i; - size_t out_size; - void *out_buf; - flb_sds_t index_key; - struct cfl_kvlist *entry_kvlist; - - index_key = flb_sds_create_size(strlen(key) + 32); - if (index_key == NULL) { - return NULL; - } - - if (flb_sds_printf(&index_key, "%p:%s", (void *) root, key) == NULL) { - flb_sds_destroy(index_key); - return NULL; - } - - for (i = 0; i < flb_sds_len(index_key); i++) { - index_key[i] = tolower((unsigned char) index_key[i]); - } - - ret = flb_hash_table_get(index, index_key, flb_sds_len(index_key), - &out_buf, &out_size); - if (ret >= 0) { - flb_sds_destroy(index_key); - return (struct cfl_kvlist *) out_buf; - } - - entry_kvlist = cfl_kvlist_create(); - if (entry_kvlist == NULL) { - flb_sds_destroy(index_key); - return NULL; - } - - ret = cfl_kvlist_insert_kvlist(root, key, entry_kvlist); - if (ret != 0) { - cfl_kvlist_destroy(entry_kvlist); - flb_sds_destroy(index_key); - return NULL; - } - - /* the kvlist is owned by root, the index only references it */ - ret = flb_hash_table_add(index, index_key, flb_sds_len(index_key), - entry_kvlist, 0); - flb_sds_destroy(index_key); - if (ret == -1) { - return NULL; - } - - return entry_kvlist; -} - static uint64_t compute_metric_hash(struct cmt_map *map, struct cmt_metric *sample) { struct cfl_list *head; @@ -465,31 +391,10 @@ static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct flb_hash_table *kvlist_index, struct cmt_map *map) { - struct cfl_kvlist *otlp_root; - struct cfl_kvlist *metrics_root; - struct cfl_kvlist *type_root; - - if (cmt == NULL || map == NULL || map->opts == NULL || map->opts->fqname == NULL) { - return NULL; - } - - otlp_root = get_or_create_indexed_kvlist(kvlist_index, cmt->external_metadata, "otlp"); - if (otlp_root == NULL) { - return NULL; + if (!map->otlp_metadata) { + map->otlp_metadata = cfl_kvlist_create(); } - - metrics_root = get_or_create_indexed_kvlist(kvlist_index, otlp_root, "metrics"); - if (metrics_root == NULL) { - return NULL; - } - - type_root = get_or_create_indexed_kvlist(kvlist_index, metrics_root, - map_type_to_key(map->type)); - if (type_root == NULL) { - return NULL; - } - - return get_or_create_indexed_kvlist(kvlist_index, type_root, map->opts->fqname); + return map->otlp_metadata; } static struct cfl_kvlist *get_or_create_data_point_metadata_context( @@ -499,30 +404,81 @@ static struct cfl_kvlist *get_or_create_data_point_metadata_context( struct cmt_metric *sample, uint64_t timestamp) { - char key[128]; - struct cfl_kvlist *metric_context; - struct cfl_kvlist *datapoints_context; - - if (sample != NULL && sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { + if (sample == NULL) { + return NULL; + } + if (sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { sample->hash = compute_metric_hash(map, sample); } + if (sample->otlp_metadata == NULL) { + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata && cmt_otlp_capture_labels(map, sample) != 0) { + return NULL; + } + } + return sample->otlp_metadata; +} - metric_context = get_or_create_metric_metadata_context(cmt, kvlist_index, map); - if (metric_context == NULL) { +/* + * Release the per-sample storage owned by the static (label-less) sample of a + * map and mark it unset. The static sample is embedded in the map, it must + * never be released with cmt_map_metric_destroy(). + */ +static void reset_static_otlp_sample(struct cmt_map *map) +{ + struct cmt_metric *sample; + + sample = &map->metric; + + free(sample->hist_buckets); + free(sample->exp_hist_positive_buckets); + free(sample->exp_hist_negative_buckets); + free(sample->sum_quantiles); + cfl_kvlist_destroy(sample->otlp_metadata); + + sample->hist_buckets = NULL; + sample->exp_hist_positive_buckets = NULL; + sample->exp_hist_negative_buckets = NULL; + sample->sum_quantiles = NULL; + sample->sum_quantiles_count = 0; + sample->otlp_metadata = NULL; + + map->metric_static_set = CMT_FALSE; +} + +/* + * Get (or create) the sample for the given label values. Repeated data points + * of the same series within one request update the same sample, the last data + * point wins. The typed OTLP metadata of the sample is rebuilt so it always + * describes the last data point written to it. + */ +static struct cmt_metric *get_otlp_sample(struct cmt_map *map, int count, char **values) +{ + struct cmt_metric *sample; + + sample = cmt_map_metric_get(map->opts, map, count, values, CMT_TRUE); + if (sample == NULL) { return NULL; } - datapoints_context = get_or_create_indexed_kvlist(kvlist_index, metric_context, - "datapoints"); - if (datapoints_context == NULL) { - return NULL; + if (sample->otlp_metadata != NULL) { + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = NULL; } - snprintf(key, sizeof(key) - 1, "%llx:%llu", - (unsigned long long) (sample != NULL ? sample->hash : 0), - (unsigned long long) timestamp); + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata == NULL || + cmt_otlp_capture_labels(map, sample) != 0) { + if (sample == &map->metric) { + reset_static_otlp_sample(map); + } + else { + cmt_map_metric_destroy(sample); + } + return NULL; + } - return get_or_create_indexed_kvlist(kvlist_index, datapoints_context, key); + return sample; } static int object_to_sds(msgpack_object *obj, flb_sds_t *out) @@ -1123,6 +1079,7 @@ static void append_common_datapoint_metadata(struct cfl_kvlist *point_metadata, msgpack_object_map *point_map) { int result; + struct cfl_kvlist *attributes; uint64_t start_time_unix_nano; uint64_t flags; msgpack_object *exemplars_object; @@ -1131,6 +1088,16 @@ static void append_common_datapoint_metadata(struct cfl_kvlist *point_metadata, return; } + result = flb_otel_utils_find_map_entry_by_key(point_map, "attributes", 0, FLB_TRUE); + attributes = cfl_kvlist_create(); + if (attributes != NULL) { + if ((result >= 0 && flb_otel_utils_clone_kvlist_from_otlp_json_array( + attributes, &point_map->ptr[result].val) != 0) || + cfl_kvlist_insert_kvlist(point_metadata, "attributes", attributes) != 0) { + cfl_kvlist_destroy(attributes); + } + } + result = parse_optional_datapoint_u64(point_map, "startTimeUnixNano", &start_time_unix_nano); @@ -1577,6 +1544,7 @@ static int process_metric_gauge_data_points(struct cmt *context, return result; } + gauge->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -1591,11 +1559,8 @@ static int process_metric_gauge_data_points(struct cmt *context, } } - sample = cmt_map_metric_get(&gauge->opts, - gauge->map, - point_label_count, - (char **) point_label_values, - CMT_TRUE); + sample = get_otlp_sample(gauge->map, point_label_count, + (char **) point_label_values); if (sample != NULL) { if (number_value_case == CMT_METRIC_VALUE_INT64) { cmt_metric_set_int64(sample, timestamp, int_value); @@ -1771,6 +1736,7 @@ static int process_metric_sum_data_points(struct cmt *context, } counter->aggregation_type = aggregation_type; + counter->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -1785,11 +1751,8 @@ static int process_metric_sum_data_points(struct cmt *context, } } - sample = cmt_map_metric_get(&counter->opts, - counter->map, - point_label_count, - (char **) point_label_values, - CMT_TRUE); + sample = get_otlp_sample(counter->map, point_label_count, + (char **) point_label_values); if (sample != NULL) { if (number_value_case == CMT_METRIC_VALUE_INT64) { if (allow_reset == FLB_FALSE && int_value < 0) { @@ -2176,6 +2139,7 @@ static int process_metric_histogram_data_points(struct cmt *context, uint64_t timestamp; uint64_t count; size_t bucket_counts_count; + size_t bucket_index; size_t metric_bound_count; size_t point_bound_count; double sum; @@ -2466,6 +2430,7 @@ static int process_metric_histogram_data_points(struct cmt *context, } histogram->aggregation_type = aggregation_type; + histogram->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -2492,19 +2457,21 @@ static int process_metric_histogram_data_points(struct cmt *context, } } - cmt_histogram_set_default(histogram, - timestamp, - bucket_counts, - sum, - count, - point_label_count, - (char **) point_label_values); + sample = get_otlp_sample(histogram->map, point_label_count, + (char **) point_label_values); + if (sample != NULL) { + /* CMetrics stores cumulative buckets, including +Inf. */ + for (bucket_index = 1; bucket_index < bucket_counts_count; bucket_index++) { + bucket_counts[bucket_index] += bucket_counts[bucket_index - 1]; + } + /* a repeated series replaces the buckets of the previous point */ + free(sample->hist_buckets); + sample->hist_buckets = bucket_counts; + bucket_counts = NULL; + cmt_metric_hist_sum_set(sample, timestamp, sum); + cmt_metric_hist_count_set(sample, timestamp, count); + } - sample = cmt_map_metric_get(&histogram->opts, - histogram->map, - point_label_count, - (char **) point_label_values, - CMT_FALSE); if (sample != NULL) { point_metadata = get_or_create_data_point_metadata_context(context, kvlist_index, @@ -2544,6 +2511,7 @@ static int process_metric_summary_data_points(struct cmt *context, msgpack_object *name_object, msgpack_object_array *data_points) { + int quantile_index; int index; int result; int metric_initialized; @@ -2758,6 +2726,7 @@ static int process_metric_summary_data_points(struct cmt *context, return result; } + summary->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -2784,19 +2753,34 @@ static int process_metric_summary_data_points(struct cmt *context, } } - cmt_summary_set_default(summary, - timestamp, - point_quantile_values, - sum, - count, - point_label_count, - (char **) point_label_values); + sample = get_otlp_sample(summary->map, point_label_count, + (char **) point_label_values); + if (sample != NULL) { + /* a repeated series reuses the quantile storage of the sample */ + if (sample->sum_quantiles != NULL && + sample->sum_quantiles_count != summary->quantiles_count) { + free(sample->sum_quantiles); + sample->sum_quantiles = NULL; + } + if (sample->sum_quantiles == NULL) { + sample->sum_quantiles = calloc(summary->quantiles_count ? + summary->quantiles_count : 1, + sizeof(uint64_t)); + } + if (sample->sum_quantiles == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + flb_free(point_quantile_values); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + sample->sum_quantiles_count = summary->quantiles_count; + cmt_atomic_store(&sample->sum_quantiles_set, CMT_TRUE); + for (quantile_index = 0; quantile_index < (int) summary->quantiles_count; quantile_index++) { + cmt_summary_quantile_set(sample, timestamp, quantile_index, point_quantile_values[quantile_index]); + } + cmt_summary_sum_set(sample, timestamp, sum); + cmt_summary_count_set(sample, timestamp, count); + } - sample = cmt_map_metric_get(&summary->opts, - summary->map, - point_label_count, - (char **) point_label_values, - CMT_FALSE); if (sample != NULL) { point_metadata = get_or_create_data_point_metadata_context(context, kvlist_index, @@ -3144,6 +3128,7 @@ static int process_metric_exponential_histogram_data_points( } exp_histogram->aggregation_type = aggregation_type; + exp_histogram->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -3163,34 +3148,28 @@ static int process_metric_exponential_histogram_data_points( } } - result = cmt_exp_histogram_set_default(exp_histogram, - timestamp, - scale, - zero_count, - zero_threshold, - positive_offset, - positive_count, - positive_counts, - negative_offset, - negative_count, - negative_counts, - sum_set, - sum, - count, - point_label_count, - (char **) point_label_values); - if (result != 0) { - destroy_label_arrays(point_label_count, NULL, point_label_values); - flb_free(positive_counts); - flb_free(negative_counts); - return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + sample = get_otlp_sample(exp_histogram->map, point_label_count, + (char **) point_label_values); + if (sample != NULL) { + sample->exp_hist_scale = scale; + sample->exp_hist_zero_count = zero_count; + sample->exp_hist_zero_threshold = zero_threshold; + sample->exp_hist_positive_offset = positive_offset; + sample->exp_hist_negative_offset = negative_offset; + sample->exp_hist_positive_count = positive_count; + sample->exp_hist_negative_count = negative_count; + /* a repeated series replaces the buckets of the previous point */ + free(sample->exp_hist_positive_buckets); + free(sample->exp_hist_negative_buckets); + sample->exp_hist_positive_buckets = positive_counts; + sample->exp_hist_negative_buckets = negative_counts; + positive_counts = NULL; + negative_counts = NULL; + cmt_metric_set_exp_hist_count(sample, count); + cmt_metric_set_exp_hist_sum(sample, sum_set, sum); + cmt_metric_set_timestamp(sample, timestamp); } - sample = cmt_map_metric_get(&exp_histogram->opts, - exp_histogram->map, - point_label_count, - (char **) point_label_values, - CMT_FALSE); if (sample != NULL) { point_metadata = get_or_create_data_point_metadata_context(context, kvlist_index, diff --git a/src/opentelemetry/flb_opentelemetry_otlp_json.c b/src/opentelemetry/flb_opentelemetry_otlp_json.c index 150dcff2aad..316c2393a3b 100644 --- a/src/opentelemetry/flb_opentelemetry_otlp_json.c +++ b/src/opentelemetry/flb_opentelemetry_otlp_json.c @@ -17,6 +17,7 @@ * limitations under the License. */ +#include #include #include #include @@ -1541,6 +1542,25 @@ static struct flb_json_mut_val *log_record_to_json(struct flb_json_mut_doc *doc, } } + if (otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP && + otlp_uint64_field_value(&otlp_metadata->via.map, "dropped_attributes_count", ×tamp) == 0 && + !flb_json_mut_obj_add_uint(doc, record, "droppedAttributesCount", timestamp)) { + return NULL; + } + field = otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP ? + msgpack_map_get_object(&otlp_metadata->via.map, "trace_flags") : NULL; + if (field != NULL && field->type == MSGPACK_OBJECT_POSITIVE_INTEGER && + !flb_json_mut_obj_add_uint(doc, record, "flags", field->via.u64)) { + return NULL; + } + + field = otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP ? + msgpack_map_get_object(&otlp_metadata->via.map, "event_name") : NULL; + if (field != NULL && field->type == MSGPACK_OBJECT_STR && + !flb_json_mut_obj_add_strncpy(doc, record, "eventName", field->via.str.ptr, field->via.str.size)) { + return NULL; + } + body = extract_log_body_value(doc, event->body, logs_body_keys, @@ -2068,6 +2088,10 @@ static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, return NULL; } + if (map->otlp_metadata) { + return map->otlp_metadata; + } + otlp_root = fetch_metadata_kvlist_key(cmt->external_metadata, "otlp"); metrics_root = fetch_metadata_kvlist_key(otlp_root, "metrics"); type_root = fetch_metadata_kvlist_key(metrics_root, metrics_map_type_to_key(map->type)); @@ -2087,6 +2111,10 @@ static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, struct cfl_kvlist *metric_context; struct cfl_kvlist *datapoints_context; + if (sample != NULL && sample->otlp_metadata != NULL) { + return sample->otlp_metadata; + } + metric_context = get_metric_otlp_metadata_context(cmt, map); datapoints_context = fetch_metadata_kvlist_key(metric_context, "datapoints"); @@ -2131,7 +2159,10 @@ static size_t resolve_target_scope_index(struct cmt *cmt, struct cfl_kvlist *metric_context; metric_context = get_metric_otlp_metadata_context(cmt, map); - metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + metadata = fetch_metadata_kvlist_key(metric_context, "routing"); + if (metadata == NULL) { + metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + } if (metadata == NULL) { return 0; @@ -2397,6 +2428,7 @@ static int add_data_point_common_fields(struct flb_json_mut_doc *doc, struct cmt_map_label *label_name; struct cmt_map_label *label_value; struct cfl_kvlist *metadata; + struct cfl_kvlist *typed_attributes; struct cfl_variant *variant; struct cfl_array *array; struct flb_json_mut_val *attributes; @@ -2404,7 +2436,13 @@ static int add_data_point_common_fields(struct flb_json_mut_doc *doc, struct flb_json_mut_val *value; struct flb_json_mut_val *exemplars; - attributes = flb_json_mut_arr(doc); + metadata = get_data_point_otlp_metadata_context(cmt, map, sample); + typed_attributes = fetch_metadata_kvlist_key(metadata, "attributes"); + if (!cmt_otlp_attributes_match_labels(map, sample)) { + typed_attributes = NULL; + } + attributes = typed_attributes ? cfl_kvlist_to_otlp_kv_array(doc, typed_attributes) : + flb_json_mut_arr(doc); if (attributes == NULL) { return -1; } @@ -2429,6 +2467,7 @@ static int add_data_point_common_fields(struct flb_json_mut_doc *doc, label_index = 0; label_name = cfl_list_entry_first(&map->label_keys, struct cmt_map_label, _head); + if (typed_attributes == NULL) { cfl_list_foreach(head, &sample->labels) { label_value = cfl_list_entry(head, struct cmt_map_label, _head); attribute = flb_json_mut_obj(doc); @@ -2454,6 +2493,8 @@ static int add_data_point_common_fields(struct flb_json_mut_doc *doc, &map->label_keys); } + } + if (flb_json_mut_arr_size(attributes) > 0 && !flb_json_mut_obj_add_val(doc, point, "attributes", attributes)) { return -1; @@ -2697,7 +2738,7 @@ static struct flb_json_mut_val *create_histogram_data_point_json(struct flb_json int length; length = snprintf(buffer, sizeof(buffer), "%" PRIu64, - sample->hist_buckets[index]); + cmt_metric_hist_get_bucket_count(sample, index)); if (length <= 0 || (size_t) length >= sizeof(buffer) || !flb_json_mut_arr_add_strncpy(doc, bucket_counts, buffer, length)) { return NULL; @@ -3628,8 +3669,7 @@ static struct flb_json_mut_val *create_trace_span_json(struct flb_json_mut_doc * return NULL; } - if (span->kind >= 0 && - !flb_json_mut_obj_add_int(doc, json, "kind", span->kind)) { + if (!flb_json_mut_obj_add_int(doc, json, "kind", span->kind)) { return NULL; } diff --git a/src/opentelemetry/flb_opentelemetry_traces.c b/src/opentelemetry/flb_opentelemetry_traces.c index ad5f3118b93..56b4e185e68 100644 --- a/src/opentelemetry/flb_opentelemetry_traces.c +++ b/src/opentelemetry/flb_opentelemetry_traces.c @@ -26,194 +26,21 @@ #include -static int process_attribute(struct ctrace_attributes *attr, - msgpack_object *key, msgpack_object *value, int type) -{ - int ret; - char *key_str; - char *value_str; - int64_t value_int; - double value_double; - int value_bool; - - if (key->type != MSGPACK_OBJECT_STR) { - return -1; - } - - /* temporary buffer for key since it needs to be NULL terminated */ - key_str = flb_sds_create_len(key->via.str.ptr, key->via.str.size); - if (key_str == NULL) { - return -1; - } - - /* - * the value of 'type' is set by the JSON wrapped value, we need to to convert it - * since msgpack_object *value is always a string - */ - switch (type) { - case MSGPACK_OBJECT_STR: - if (value->type != MSGPACK_OBJECT_STR) { - flb_sds_destroy(key_str); - return -1; - } - - value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); - if (value_str == NULL) { - flb_sds_destroy(key_str); - return -1; - } - - ret = ctr_attributes_set_string(attr, key_str, value_str); - flb_sds_destroy(value_str); - break; - case MSGPACK_OBJECT_POSITIVE_INTEGER: - case MSGPACK_OBJECT_NEGATIVE_INTEGER: - if (value->type == MSGPACK_OBJECT_STR) { - value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); - if (value_str == NULL) { - flb_sds_destroy(key_str); - return -1; - } - value_int = strtoll(value_str, NULL, 10); - flb_sds_destroy(value_str); - } - else if (value->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - value->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - value_int = value->via.i64; - } - else { - flb_sds_destroy(key_str); - return -1; - } - - ret = ctr_attributes_set_int64(attr, key_str, value_int); - break; - case MSGPACK_OBJECT_FLOAT32: - case MSGPACK_OBJECT_FLOAT64: - if (value->type == MSGPACK_OBJECT_STR) { - value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); - if (value_str == NULL) { - flb_sds_destroy(key_str); - return -1; - } - value_double = strtod(value_str, NULL); - flb_sds_destroy(value_str); - } - else if (value->type == MSGPACK_OBJECT_FLOAT32 || - value->type == MSGPACK_OBJECT_FLOAT64) { - value_double = value->via.f64; - } - else { - flb_sds_destroy(key_str); - return -1; - } - - ret = ctr_attributes_set_double(attr, key_str, value_double); - break; - case MSGPACK_OBJECT_BOOLEAN: - if (value->type != MSGPACK_OBJECT_BOOLEAN) { - flb_sds_destroy(key_str); - return -1; - } - - value_bool = value->via.boolean; - ret = ctr_attributes_set_bool(attr, key_str, value_bool); - break; - case MSGPACK_OBJECT_ARRAY: - /* - * The less fun part (OTel JSON encoding), the value can be an array and - * only allows values such as string, bool, int64, double. I am glad this - * don't support nested arrays or maps. - */ - ret = 0; - break; - default: - flb_sds_destroy(key_str); - return -1; - } - - flb_sds_destroy(key_str); - return ret; -} - -static int process_resource_unwrap_attribute(msgpack_object *attr, - msgpack_object *out_key, - msgpack_object *out_value, int *out_value_type) -{ - int ret; - int type; - msgpack_object key; - msgpack_object val; - msgpack_object *real_value; - - if (attr->type != MSGPACK_OBJECT_MAP) { - return -1; - } - - ret = flb_otel_utils_find_map_entry_by_key(&attr->via.map, "key", 0, FLB_TRUE); - if (ret == -1) { - return -1; - } - - key = attr->via.map.ptr[ret].val; - if (key.type != MSGPACK_OBJECT_STR) { - return -1; - } - - ret = flb_otel_utils_find_map_entry_by_key(&attr->via.map, "value", 0, FLB_TRUE); - if (ret == -1) { - return -1; - } - - val = attr->via.map.ptr[ret].val; - - ret = flb_otel_utils_json_payload_get_wrapped_value(&val, &real_value, &type); - if (ret != 0) { - return -1; - } - - *out_key = key; - *out_value = *real_value; - *out_value_type = type; - - return 0; -} - -/* - * Convert a list of attributes in msgpack format to a cfl attributes by - * unwrapping JSON encoded value types. - */ +/* Use the shared AnyValue decoder for scalars, bytes, arrays and nested maps. */ static struct ctrace_attributes *convert_attributes(msgpack_object *attributes, const char *log_context) { - int i; - int ret; - int value_type; - msgpack_object key; - msgpack_object value; struct ctrace_attributes *attr; attr = ctr_attributes_create(); - if (attr == NULL) { + if (!attr) { return NULL; } - - for (i = 0; i < attributes->via.array.size; i++) { - ret = process_resource_unwrap_attribute(&attributes->via.array.ptr[i], - &key, &value, &value_type); - if (ret == -1) { - flb_warn("found invalid %s attribute, skipping", log_context); - continue; - } - - /* set attribute */ - ret = process_attribute(attr, &key, &value, value_type); - if (ret == -1) { - flb_warn("failed to set %s attribute, skipping", log_context); - continue; - } + if (flb_otel_utils_clone_kvlist_from_otlp_json_array(attr->kv, attributes) != 0) { + flb_warn("failed to decode %s attributes", log_context); + ctr_attributes_destroy(attr); + return NULL; } - return attr; } @@ -854,8 +681,16 @@ static int process_spans(struct ctrace *ctr, /* kind */ ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "kind", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - ctr_span_kind_set(ctr_span, span.via.map.ptr[ret].val.via.u64); + if (ret >= 0) { + /* Proto3 enums preserve unknown numeric values within int32. */ + if (span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER && + span.via.map.ptr[ret].val.via.u64 <= INT32_MAX) { + ctr_span->kind = (int32_t) span.via.map.ptr[ret].val.via.u64; + } + else if (span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + span.via.map.ptr[ret].val.via.i64 >= INT32_MIN) { + ctr_span->kind = (int32_t) span.via.map.ptr[ret].val.via.i64; + } } /* attributes */ diff --git a/src/opentelemetry/flb_opentelemetry_utils.c b/src/opentelemetry/flb_opentelemetry_utils.c index f6e97620c3b..2eea40a04db 100644 --- a/src/opentelemetry/flb_opentelemetry_utils.c +++ b/src/opentelemetry/flb_opentelemetry_utils.c @@ -26,6 +26,76 @@ #include #include +#include + +/* + * OTLP/JSON encodes bytesValue as base64 (standard or URL safe alphabet, with + * or without padding). Decode it into a newly allocated buffer owned by the + * caller. Returns 0 on success and -1 for invalid input or allocation errors. + */ +static int otlp_json_bytes_decode(const char *src, size_t src_size, + unsigned char **out_buf, size_t *out_size) +{ + int ret; + char c; + char *normalized; + size_t index; + size_t padded_size; + size_t decoded_size; + unsigned char *decoded; + + *out_buf = NULL; + *out_size = 0; + + if (src_size % 4 == 1) { + return -1; + } + + padded_size = src_size + ((4 - (src_size % 4)) % 4); + + normalized = flb_malloc(padded_size + 1); + if (normalized == NULL) { + flb_errno(); + return -1; + } + + for (index = 0; index < src_size; index++) { + c = src[index]; + if (c == '-') { + c = '+'; + } + else if (c == '_') { + c = '/'; + } + normalized[index] = c; + } + + for (; index < padded_size; index++) { + normalized[index] = '='; + } + normalized[padded_size] = '\0'; + + decoded = flb_malloc((padded_size / 4) * 3 + 1); + if (decoded == NULL) { + flb_errno(); + flb_free(normalized); + return -1; + } + + ret = flb_base64_decode(decoded, (padded_size / 4) * 3 + 1, &decoded_size, + (unsigned char *) normalized, padded_size); + flb_free(normalized); + + if (ret != 0) { + flb_free(decoded); + return -1; + } + + *out_buf = decoded; + *out_size = decoded_size; + + return 0; +} int flb_otel_utils_find_map_entry_by_key(msgpack_object_map *map, char *key, @@ -132,7 +202,8 @@ int flb_otel_utils_json_payload_get_wrapped_value(msgpack_object *wrapper, internal_type = MSGPACK_OBJECT_FLOAT; } else if (strncasecmp(kv_key->ptr, "bytesValue", kv_key->size) == 0) { - if (kv_value->type != MSGPACK_OBJECT_BIN) { + if (kv_value->type != MSGPACK_OBJECT_BIN && + kv_value->type != MSGPACK_OBJECT_STR) { /* If the value is not binary, we cannot process it */ return -2; } @@ -300,6 +371,8 @@ int flb_otel_utils_json_payload_append_unwrapped_value( char temporary_buffer[33]; int unwrap_value; int result; + size_t bytes_size; + unsigned char *bytes; msgpack_object *value; int type; @@ -344,7 +417,19 @@ int flb_otel_utils_json_payload_append_unwrapped_value( unwrap_value = FLB_TRUE; } else if (type == MSGPACK_OBJECT_BIN) { - unwrap_value = FLB_TRUE; + if (value->type == MSGPACK_OBJECT_STR) { + if (otlp_json_bytes_decode(value->via.str.ptr, value->via.str.size, + &bytes, &bytes_size) != 0) { + return -2; + } + + result = flb_log_event_encoder_append_binary(encoder, target_field, + (char *) bytes, bytes_size); + flb_free(bytes); + } + else { + unwrap_value = FLB_TRUE; + } } else if (type == MSGPACK_OBJECT_ARRAY) { @@ -694,58 +779,6 @@ uint64_t flb_otel_utils_convert_string_number_to_u64(char *str, size_t len) return val; } -static int parse_u64_from_numeric_string(const char *str, - size_t len, - uint64_t *value) -{ - size_t index; - - if (str == NULL || value == NULL || len == 0) { - return -1; - } - - if (str[0] == '-' || len > 31) { - return -1; - } - - for (index = 0; index < len; index++) { - if (!isdigit((unsigned char) str[index])) { - return -1; - } - } - - *value = flb_otel_utils_convert_string_number_to_u64((char *) str, len); - - return 0; -} - -static int parse_u64_from_msgpack_object(msgpack_object *object, uint64_t *value) -{ - if (object == NULL || value == NULL) { - return -1; - } - - if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - *value = object->via.u64; - return 0; - } - else if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - if (object->via.i64 < 0) { - return -1; - } - - *value = (uint64_t) object->via.i64; - return 0; - } - else if (object->type == MSGPACK_OBJECT_STR) { - return parse_u64_from_numeric_string(object->via.str.ptr, - object->via.str.size, - value); - } - - return -1; -} - static int parse_double_from_msgpack_object(msgpack_object *object, double *value) { char *end; @@ -790,6 +823,9 @@ static int parse_double_from_msgpack_object(msgpack_object *object, double *valu return 0; } +static struct cfl_variant *convert_otlp_anyvalue_wrapper_to_cfl_variant( + msgpack_object *wrapper_object); + static int clone_kvlist_entry_from_otlp_json(struct cfl_kvlist *target, msgpack_object *entry_object) { @@ -821,7 +857,7 @@ static int clone_kvlist_entry_from_otlp_json(struct cfl_kvlist *target, return -1; } - value_instance = flb_otel_utils_msgpack_object_to_cfl_variant(value_object); + value_instance = convert_otlp_anyvalue_wrapper_to_cfl_variant(value_object); if (value_instance == NULL) { return -1; } @@ -837,12 +873,58 @@ static int clone_kvlist_entry_from_otlp_json(struct cfl_kvlist *target, return 0; } +static int parse_i64_from_msgpack_object(msgpack_object *object, int64_t *value) +{ + uint64_t magnitude = 0; + uint64_t limit; + size_t index = 0; + unsigned int digit; + int negative = 0; + + if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *value = object->via.i64; + return 0; + } + if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (object->via.u64 > INT64_MAX) { + return -1; + } + *value = (int64_t) object->via.u64; + return 0; + } + if (object->type != MSGPACK_OBJECT_STR || object->via.str.size == 0) { + return -1; + } + if (object->via.str.ptr[0] == '-') { + negative = 1; + index = 1; + } + if (index == object->via.str.size) { + return -1; + } + limit = (uint64_t) INT64_MAX + negative; + for (; index < object->via.str.size; index++) { + if (object->via.str.ptr[index] < '0' || object->via.str.ptr[index] > '9') { + return -1; + } + digit = object->via.str.ptr[index] - '0'; + if (magnitude > (limit - digit) / 10) { + return -1; + } + magnitude = magnitude * 10 + digit; + } + *value = negative ? (magnitude == (uint64_t) INT64_MAX + 1 ? INT64_MIN : + -(int64_t) magnitude) : (int64_t) magnitude; + return 0; +} + static struct cfl_variant *convert_otlp_anyvalue_wrapper_to_cfl_variant( msgpack_object *wrapper_object) { int value_type; int64_t signed_value; - uint64_t unsigned_value; + unsigned char *bytes; + size_t bytes_size; double double_value; size_t index; msgpack_object *value_object; @@ -883,19 +965,10 @@ static struct cfl_variant *convert_otlp_anyvalue_wrapper_to_cfl_variant( return cfl_variant_create_from_bool(value_object->via.boolean); } else if (value_type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - if (parse_u64_from_msgpack_object(value_object, &unsigned_value) != 0) { + if (parse_i64_from_msgpack_object(value_object, &signed_value) != 0) { return NULL; } - - if (value_object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER || - (value_object->type == MSGPACK_OBJECT_STR && - value_object->via.str.size > 0 && - value_object->via.str.ptr[0] == '-')) { - signed_value = (int64_t) unsigned_value; - return cfl_variant_create_from_int64(signed_value); - } - - return cfl_variant_create_from_uint64(unsigned_value); + return cfl_variant_create_from_int64(signed_value); } else if (value_type == MSGPACK_OBJECT_FLOAT) { if (parse_double_from_msgpack_object(value_object, &double_value) != 0) { @@ -905,6 +978,17 @@ static struct cfl_variant *convert_otlp_anyvalue_wrapper_to_cfl_variant( return cfl_variant_create_from_double(double_value); } else if (value_type == MSGPACK_OBJECT_BIN) { + if (value_object->type == MSGPACK_OBJECT_STR) { + if (otlp_json_bytes_decode(value_object->via.str.ptr, + value_object->via.str.size, + &bytes, &bytes_size) != 0) { + return NULL; + } + result_variant = cfl_variant_create_from_bytes((char *) bytes, bytes_size, + CFL_FALSE); + flb_free(bytes); + return result_variant; + } if (value_object->type != MSGPACK_OBJECT_BIN) { return NULL; } @@ -1134,6 +1218,8 @@ int flb_otel_utils_clone_kvlist_from_otlp_json_array(struct cfl_kvlist *target, msgpack_object *attributes_object) { size_t index; + size_t invalid_count; + size_t first_invalid; msgpack_object_array *attributes_array; if (target == NULL || attributes_object == NULL) { @@ -1145,13 +1231,26 @@ int flb_otel_utils_clone_kvlist_from_otlp_json_array(struct cfl_kvlist *target, } attributes_array = &attributes_object->via.array; + invalid_count = 0; + first_invalid = 0; for (index = 0 ; index < attributes_array->size ; index++) { if (clone_kvlist_entry_from_otlp_json(target, &attributes_array->ptr[index]) != 0) { - return -1; + /* An invalid attribute must not discard its healthy siblings. */ + if (invalid_count == 0) { + first_invalid = index; + } + invalid_count++; } } + /* report once per attribute list to avoid flooding the log */ + if (invalid_count > 0) { + flb_warn("ignoring %zu invalid OTLP JSON attribute(s) out of %zu " + "(first at index %zu)", + invalid_count, (size_t) attributes_array->size, first_invalid); + } + return 0; } From ddff18c98ea8b4aa7a93db62cc5cb51e89100953 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 10/61] out_opentelemetry: export event name and dropped attributes count Signed-off-by: Eduardo Silva --- .../out_opentelemetry/opentelemetry_logs.c | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/plugins/out_opentelemetry/opentelemetry_logs.c b/plugins/out_opentelemetry/opentelemetry_logs.c index e1366a995bb..f3a9835ece6 100644 --- a/plugins/out_opentelemetry/opentelemetry_logs.c +++ b/plugins/out_opentelemetry/opentelemetry_logs.c @@ -175,6 +175,32 @@ static msgpack_object *msgpack_map_get_object(msgpack_object_map *map, return &map->ptr[index].val; } +static uint32_t dropped_attribute_count(msgpack_object *object) +{ + msgpack_object *field; + + if (!object || object->type != MSGPACK_OBJECT_MAP) { + return 0; + } + + field = msgpack_map_get_object(&object->via.map, "dropped_attributes_count"); + if (field && field->type == MSGPACK_OBJECT_POSITIVE_INTEGER && + field->via.u64 <= UINT32_MAX) { + return (uint32_t) field->via.u64; + } + + return 0; +} + +static uint32_t group_dropped_attribute_count(msgpack_object *body, const char *kind) +{ + if (!body || body->type != MSGPACK_OBJECT_MAP) { + return 0; + } + + return dropped_attribute_count(msgpack_map_get_object(&body->via.map, kind)); +} + static uint64_t msgpack_object_hash(msgpack_object *object) { uint64_t hash; @@ -570,6 +596,8 @@ static int append_v1_logs_metadata_and_fields(struct opentelemetry_context *ctx, int trace_flags_set = FLB_FALSE; size_t attr_count = 0; struct flb_ra_value *ra_val; + msgpack_object *otlp_metadata; + msgpack_object *event_name; Opentelemetry__Proto__Common__V1__KeyValue **attrs = NULL; if (ctx == NULL || event == NULL || log_record == NULL) { @@ -824,6 +852,25 @@ static int append_v1_logs_metadata_and_fields(struct opentelemetry_context *ctx, } } + if (event->metadata && event->metadata->type == MSGPACK_OBJECT_MAP) { + otlp_metadata = msgpack_map_get_object(&event->metadata->via.map, "otlp"); + log_record->dropped_attributes_count = dropped_attribute_count(otlp_metadata); + + event_name = NULL; + if (otlp_metadata && otlp_metadata->type == MSGPACK_OBJECT_MAP) { + event_name = msgpack_map_get_object(&otlp_metadata->via.map, "event_name"); + } + + if (event_name && event_name->type == MSGPACK_OBJECT_STR) { + log_record->event_name = flb_strndup(event_name->via.str.ptr, + event_name->via.str.size); + if (!log_record->event_name) { + flb_errno(); + return -1; + } + } + } + return 0; } @@ -851,6 +898,9 @@ static void free_log_records(Opentelemetry__Proto__Logs__V1__LogRecord **logs, s if (log->severity_text != NULL && log->severity_text != protobuf_c_empty_string) { flb_free(log->severity_text); } + if (log->event_name != NULL && log->event_name != protobuf_c_empty_string) { + flb_free(log->event_name); + } if (log->span_id.data != NULL) { flb_free(log->span_id.data); } @@ -1355,6 +1405,8 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, /* group body: $resource['attributes'] */ set_resource_attributes(ctx->ra_resource_attr, event.body, resource_log->resource); + resource_log->resource->dropped_attributes_count = + group_dropped_attribute_count(event.body, "resource"); /* group body: $schema_url */ set_resource_schema_url(ctx->ra_resource_schema_url, event.body, resource_log); @@ -1476,6 +1528,8 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, /* group body: $scope['attributes'] */ set_scope_attributes(ctx->ra_scope_attr, event.body, scope_log->scope); + scope_log->scope->dropped_attributes_count = + group_dropped_attribute_count(event.body, "scope"); /* group body: $scope['schema_url'] */ set_scope_schema_url(ctx->ra_scope_schema_url, event.body, scope_log); From 633ad1c6e6e10b9682139fe8c71acc5b53e0ab44 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 11/61] processor_labels: keep the complete sample on static conversion Signed-off-by: Eduardo Silva --- plugins/processor_labels/labels.c | 33 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/plugins/processor_labels/labels.c b/plugins/processor_labels/labels.c index 7f2e8aac940..f81efc3be07 100644 --- a/plugins/processor_labels/labels.c +++ b/plugins/processor_labels/labels.c @@ -196,6 +196,7 @@ static void map_metric_destroy(struct cmt_metric *metric) free(metric->sum_quantiles); } + cfl_kvlist_destroy(metric->otlp_metadata); cfl_list_del(&metric->_head); free(metric); } @@ -806,6 +807,7 @@ int metrics_map_convert_static_metric(struct cmt_map *map, char *label_value) { struct cmt_metric *metric; + struct cfl_list labels; int result; size_t index; cfl_hash_state_t state; @@ -861,23 +863,22 @@ int metrics_map_convert_static_metric(struct cmt_map *map, } } - metric->val = map->metric.val; - - metric->hist_buckets = map->metric.hist_buckets; - metric->hist_count = map->metric.hist_count; - metric->hist_sum = map->metric.hist_sum; - - metric->sum_quantiles_set = map->metric.sum_quantiles_set; - metric->sum_quantiles = map->metric.sum_quantiles; - metric->sum_quantiles_count = map->metric.sum_quantiles_count; - metric->sum_count = map->metric.sum_count; - metric->sum_sum = map->metric.sum_sum; - - metric->timestamp = map->metric.timestamp; - - map->metric_static_set = 0; + /* Transfer the complete sample, including exact scalar types, start time, + * distributions and owned metadata. Preserve the newly allocated labels. */ + labels = metric->labels; + memcpy(metric, &map->metric, sizeof(*metric)); + metric->labels = labels; + metric->map = map; + metric->hash = hash; + metric->hash_indexed = CMT_FALSE; + cfl_list_init(&metric->_hash_head); + map->metric_static_set = CMT_FALSE; - cfl_list_add(&metric->_head, &map->metrics); + /* The static sample preceded dynamic samples before the conversion. */ + cfl_list_add_before(&metric->_head, map->metrics.next, &map->metrics); + if (map->last_metric == &map->metric) { + map->last_metric = metric; + } memset(&map->metric, 0, sizeof(struct cmt_metric)); From a2a3031a118bb12773456db60810f38d26d08ca4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 12/61] processor_opentelemetry_envelope: fix envelope list insertion Signed-off-by: Eduardo Silva --- plugins/processor_opentelemetry_envelope/otel_envelope.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/processor_opentelemetry_envelope/otel_envelope.c b/plugins/processor_opentelemetry_envelope/otel_envelope.c index 9978318e95a..d6bd625df08 100644 --- a/plugins/processor_opentelemetry_envelope/otel_envelope.c +++ b/plugins/processor_opentelemetry_envelope/otel_envelope.c @@ -159,8 +159,8 @@ static void envelop_end(struct cfl_list *list, struct flb_mp_chunk_record *activ record->cobj_metadata = NULL; record->cobj_record = NULL; - /* add the envelop before the active record */ - cfl_list_add_after(&record->_head, &active_record->_head, list); + /* Insert after the active record while updating both list directions. */ + cfl_list_add_before(&record->_head, active_record->_head.next, list); } From 07914b07e3f09b21043b4e91aaa430e655e9913e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 13/61] processor_fts: add Fluent Typed Scripting processor FTS runs Lua policies with typed access to logs, metrics and traces: exact integers, binary values, nanosecond timestamps and first-class OpenTelemetry resources, scopes and attributes. Edits are transactional per item and bounded by instruction, memory and output limits. Signed-off-by: Eduardo Silva --- cmake/plugins_options.cmake | 1 + plugins/CMakeLists.txt | 3 + plugins/processor_fts/CMakeLists.txt | 28 + plugins/processor_fts/README.md | 321 +++ plugins/processor_fts/fts.c | 197 ++ plugins/processor_fts/fts.h | 153 ++ plugins/processor_fts/fts_helpers.h.in | 15 + plugins/processor_fts/fts_log_helpers.lua | 171 ++ plugins/processor_fts/fts_logs.c | 1569 ++++++++++++ plugins/processor_fts/fts_metrics.c | 572 +++++ plugins/processor_fts/fts_proto.c | 616 +++++ plugins/processor_fts/fts_proto.h | 39 + plugins/processor_fts/fts_runtime.c | 436 ++++ plugins/processor_fts/fts_telemetry.c | 815 +++++++ plugins/processor_fts/fts_telemetry.h | 40 + .../processor_fts/fts_telemetry_helpers.lua | 571 +++++ plugins/processor_fts/fts_traces.c | 143 ++ plugins/processor_fts/fts_value_helpers.lua | 37 + plugins/processor_fts/fts_values.c | 2173 +++++++++++++++++ 19 files changed, 7900 insertions(+) create mode 100644 plugins/processor_fts/CMakeLists.txt create mode 100644 plugins/processor_fts/README.md create mode 100644 plugins/processor_fts/fts.c create mode 100644 plugins/processor_fts/fts.h create mode 100644 plugins/processor_fts/fts_helpers.h.in create mode 100644 plugins/processor_fts/fts_log_helpers.lua create mode 100644 plugins/processor_fts/fts_logs.c create mode 100644 plugins/processor_fts/fts_metrics.c create mode 100644 plugins/processor_fts/fts_proto.c create mode 100644 plugins/processor_fts/fts_proto.h create mode 100644 plugins/processor_fts/fts_runtime.c create mode 100644 plugins/processor_fts/fts_telemetry.c create mode 100644 plugins/processor_fts/fts_telemetry.h create mode 100644 plugins/processor_fts/fts_telemetry_helpers.lua create mode 100644 plugins/processor_fts/fts_traces.c create mode 100644 plugins/processor_fts/fts_value_helpers.lua create mode 100644 plugins/processor_fts/fts_values.c diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index c0829746dc6..8c26cfe9b64 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -78,6 +78,7 @@ DEFINE_OPTION(FLB_IN_EBPF "Enable Linux eBPF input plugin" # ========== DEFINE_OPTION(FLB_PROCESSOR_CONTENT_MODIFIER "Enable content modifier processor" ON) DEFINE_OPTION(FLB_PROCESSOR_CUMULATIVE_TO_DELTA "Enable cumulative to delta metrics processor" ON) +DEFINE_OPTION(FLB_PROCESSOR_FTS "Enable Fluent Typed Scripting processor" ON) DEFINE_OPTION(FLB_PROCESSOR_LABELS "Enable metrics label manipulation processor" ON) DEFINE_OPTION(FLB_PROCESSOR_METRICS_SELECTOR "Enable metrics selector processor" ON) DEFINE_OPTION(FLB_PROCESSOR_OPENTELEMETRY_ENVELOPE "Enable OpenTelemetry envelope processor" ON) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 86e19800367..61b970e7c33 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -347,6 +347,9 @@ REGISTER_IN_PLUGIN("in_random") # ========== REGISTER_PROCESSOR_PLUGIN("processor_content_modifier") REGISTER_PROCESSOR_PLUGIN("processor_cumulative_to_delta") +if(FLB_LUAJIT AND FLB_YYJSON) + REGISTER_PROCESSOR_PLUGIN("processor_fts") +endif() REGISTER_PROCESSOR_PLUGIN("processor_labels") REGISTER_PROCESSOR_PLUGIN("processor_metrics_selector") REGISTER_PROCESSOR_PLUGIN("processor_opentelemetry_envelope") diff --git a/plugins/processor_fts/CMakeLists.txt b/plugins/processor_fts/CMakeLists.txt new file mode 100644 index 00000000000..a27687dd11f --- /dev/null +++ b/plugins/processor_fts/CMakeLists.txt @@ -0,0 +1,28 @@ +# Embed the Lua helper sources as C string literals. +function(fts_embed_lua variable file) + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/${file}" content) + string(REPLACE "\\" "\\\\" content "${content}") + string(REPLACE "\"" "\\\"" content "${content}") + string(REPLACE "\n" "\\n\"\n\"" content "${content}") + set(${variable} "${content}" PARENT_SCOPE) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${file}) +endfunction() + +fts_embed_lua(FTS_VALUE_HELPERS fts_value_helpers.lua) +fts_embed_lua(FTS_LOG_HELPERS fts_log_helpers.lua) +fts_embed_lua(FTS_TELEMETRY_HELPERS fts_telemetry_helpers.lua) +configure_file(fts_helpers.h.in fts_helpers.h @ONLY) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +set(src + fts.c + fts_runtime.c + fts_values.c + fts_logs.c + fts_proto.c + fts_telemetry.c + fts_metrics.c + fts_traces.c + ) + +FLB_PLUGIN(processor_fts "${src}" "") diff --git a/plugins/processor_fts/README.md b/plugins/processor_fts/README.md new file mode 100644 index 00000000000..c626559f633 --- /dev/null +++ b/plugins/processor_fts/README.md @@ -0,0 +1,321 @@ +# Fluent Typed Scripting (FTS) processor + +FTS is a processor for writing telemetry policies in Lua. Policies get typed +access to the record and its context: integer widths, binary values, and +nanosecond timestamps are preserved, and OpenTelemetry resources, scopes, and +attributes are exposed as properties instead of raw metadata. + +FTS supports logs, metrics, and traces. Its configuration name is `fts`. + +| Signal | Callback | Called once per | +| ------- | ----------------- | --------------- | +| Logs | `process(log)` | log record | +| Metrics | `process(metric)` | metric family | +| Traces | `process(span)` | span | + +## Quick start + +```yaml +pipeline: + inputs: + - name: opentelemetry + port: 4318 + processors: + logs: + - name: fts + script: policy.lua + outputs: + - name: stdout + match: '*' + format: otlp_json +``` + +```lua +local fb = require("fluentbit") + +function process(log) + if log.body:get_string("path") == "/health" then + return fb.DROP + end + + fb.redact(log.body, {"authorization", "password", "access_token"}) + + if fb.is_otel(log) then + log.resource.attributes:set("deployment.environment.name", "production") + log.attributes:set("sanitized", true) + end +end +``` + +Use one processor entry and one script for each signal. + +## Configuration + +| Key | Default | Description | +| ------------------- | --------- | ----------- | +| `script` | | Path to the Lua policy file. Set either `script` or `code`. | +| `code` | | Inline Lua policy. Set either `script` or `code`. | +| `call` | `process` | Name of the callback function. | +| `error_mode` | `drop` | What happens to an item when its policy fails: `drop` removes it, `keep` forwards the original. | +| `instruction_limit` | `1000000` | Maximum Lua instructions per callback and during script load. Minimum `1000`. `0` is allowed only with `jit: true`. | +| `jit` | `false` | Enable the LuaJIT compiler. Requires `instruction_limit: 0`. | +| `memory_limit` | `32M` | Maximum memory for the policy: the Lua heap plus the typed values it holds. Minimum `262144`. | +| `output_limit` | `64M` | Maximum serialized size of one processed chunk. Minimum `65536`. | + +Size suffixes are decimal: `64K` is 64000 bytes. + +## Callback results + +| Return value | Effect | +| -------------------- | ------ | +| `nil` or `fb.KEEP` | Commit all changes made by the callback. | +| `fb.DROP` | Remove the item. | +| Lua error | Roll back every change made by the callback, then apply `error_mode`. | + +Changes are transactional per item. Edits to the body, metadata, timestamp, +resource, and scope are committed or rolled back together. + +If a processed chunk exceeds `output_limit`, the whole chunk is treated as +failed: `drop` discards it and `keep` forwards the original chunk. + +For privacy policies such as redaction, keep the default `error_mode: drop`. +With `keep`, an item whose policy fails is forwarded unmodified. + +## Typed values + +Bodies, attributes, and other structured fields are typed maps and arrays. + +### Map and array methods + +| Method | Applies to | Description | +| -------------------------- | ------------ | ----------- | +| `v:get(key)` | map, array | Child value, or `nil` when missing. Array indexes start at 1. | +| `v:get_string(key)` | map | String value, or `nil` when missing. Fails on another type. | +| `v:get_number(key)` | map | Numeric value as a Lua number, or `nil` when missing. | +| `v:get_bool(key)` | map | Boolean value, or `nil` when missing. | +| `v:set(key, value)` | map, array | Set a value. Replaces every entry with the same key. | +| `v:remove(key)` | map | Remove every entry with that key. | +| `v:ensure_map(key)` | map | Return the child map, creating it when missing. | +| `v:append(value)` | array | Append a value. | +| `v:keys()` | map | Lua table with the keys in order. | +| `v:len()` | map, array | Number of entries. | +| `v:type()` | any | `map`, `array`, `string`, `bytes`, `bool`, `int64`, `uint64`, `double`, or `null`. | +| `v:value()` | scalar | Lua value. 64-bit integers are returned as exact decimal strings. | +| `v:number()` | numeric | Lua number. Fails when an integer is beyond ±(2^53 − 1). | +| `v:copy()` | any | Independent, writable copy. | + +Keys match exact bytes; a dot in a key has no special meaning. + +### Constructors + +| Constructor | Creates | +| ------------------ | ------- | +| `fb.map()` | Empty map | +| `fb.array()` | Empty array | +| `fb.int64(v)` | Signed 64-bit integer from a decimal string or a safe Lua integer | +| `fb.uint64(v)` | Unsigned 64-bit integer from a decimal string or a safe Lua integer | +| `fb.bytes(s)` | Binary value | +| `fb.null` | Null value | + +Lua strings, booleans, and numbers can be passed to `set` and `append` +directly. Lua numbers are stored as doubles; use `fb.int64` or `fb.uint64` for +integers. Lua tables and `nil` are not values: use `fb.map()`, `fb.array()`, +and `fb.null`. + +### Value lifetime + +Values passed to a callback, and the views obtained from them, are valid only +during that callback. Using one later raises an error. To keep data between +callbacks, store `v:copy()`. Copies are independent: changing them never +changes a committed record. + +Lua globals persist for the lifetime of the processor instance. + +## Logs + +| Property | Description | +| ------------------------------------------- | ----------- | +| `log.body` | Record body map. Can be replaced with another map. | +| `log.metadata` | Record metadata map. | +| `log.timestamp` | `{sec, nsec}` table. Assign a new table to change the event time. | +| `log.tag` | Routing tag, read-only. | +| `log.attributes` | OpenTelemetry log attributes. | +| `log.resource.attributes`, `.schema_url` | OpenTelemetry resource. | +| `log.scope.attributes`, `.name`, `.version`, `.schema_url` | OpenTelemetry instrumentation scope. | +| `log.context.attributes`, `.metadata` | Group attributes and metadata of non-OpenTelemetry grouped logs. | + +OpenTelemetry properties exist only for logs that carry OpenTelemetry context, +for example logs received by the `opentelemetry` input. Use `fb.is_otel(log)` +to check. To give ordinary logs OpenTelemetry context, place the +`opentelemetry_envelope` processor before FTS. + +Resource, scope, and context changes apply only to the current record. Record +order is preserved. + +### Log helpers + +| Helper | Description | +| ------ | ----------- | +| `fb.is_otel(log)` | `true` when the log has OpenTelemetry context. | +| `fb.set_severity(log, level)` | Set severity text and number from a level name: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, and common aliases. | +| `fb.set_trace_context(log, trace_hex, span_hex)` | Set the trace and span IDs from hexadecimal strings. IDs must be nonzero. | +| `fb.log_field(log, name)` | Read an OpenTelemetry log field such as `severity_number`, `trace_id`, or `observed_timestamp`. | + +### Example: parse an embedded JSON message + +```lua +local fb = require("fluentbit") + +function process(log) + local text = log.body:get_string("message") + if not text or not text:match("^%s*{") then + return + end + + local ok, app = pcall(fb.parse_json, text) + if not ok then + return fb.DROP + end + + fb.redact(app, {"authorization", "password"}) + log.body = app + + if fb.is_otel(log) then + local level = app:get_string("level") + if level then + fb.set_severity(log, level) + end + + local time = app:get_string("time") + if time then + log.timestamp = fb.parse_time(time) + end + end +end +``` + +## Metrics + +`process(metric)` runs once per metric family. Resource and scope changes apply +to that family only. + +```lua +local fb = require("fluentbit") + +function process(metric) + if metric.name == "http.server.duration" then + metric.name = "http.server.request.duration" + fb.scale_metric(metric, 0.001, "s") + end + + for i = 1, metric.datapoints:len() do + metric.datapoints:get(i):get("attributes"):remove("request.id") + end +end +``` + +| Property | Description | +| --------------------------------------------- | ----------- | +| `metric.name`, `.description`, `.unit` | Family identity. | +| `metric.type` | `gauge`, `sum`, `histogram`, `exponential_histogram`, or `summary`. Only gauge and sum can be converted into each other. | +| `metric.datapoints` | Array of datapoint maps. | +| `metric.aggregation_temporality` | `unspecified`, `delta`, or `cumulative`. | +| `metric.is_monotonic` | Monotonicity of a sum. | +| `metric.resource`, `metric.scope` | Same properties as for logs. | + +Datapoint maps use OpenTelemetry field names in snake case, for example +`attributes`, `time_unix_nano`, `start_time_unix_nano`, `as_int`, `as_double`, +`count`, `sum`, `bucket_counts`, `explicit_bounds`, `quantile_values`, and +`exemplars`. Timestamps and counts are exact integers. Histogram buckets are +per-bucket counts. + +Removing a datapoint attribute can make two series identical, which fails the +policy. Use `fb.aggregate_metric` to combine them deliberately. + +### Metric helpers + +| Helper | Description | +| ------ | ----------- | +| `fb.metric_value(point)` | Scalar value of a gauge or sum datapoint. | +| `fb.set_metric_value(point, value)` | Set the scalar value of a datapoint. | +| `fb.scale_metric(metric, factor, unit)` | Multiply values by a positive factor and optionally set the unit. Applies to gauges, sums, histograms, and summaries, including bounds and exemplars. | +| `fb.aggregate_metric(metric, keys, op)` | Keep only the listed attributes and combine equal series. `op` is `sum`, or for gauges also `min`, `max`, `mean`, or `count`. | +| `fb.merge_histogram_buckets(metric, bounds)` | Merge histogram buckets into a subset of the existing bounds. | +| `fb.copy_metric(metric, name, description, unit)` | Emit a copy of the family with a new name. | +| `fb.extract_metric(metric, field, name, monotonic)` | Emit a sum derived from a histogram or summary: `count`, `sum`, or `avg`. | + +Emitted families follow the original and continue through the pipeline. They +are not passed to the callback again. Returning `fb.DROP` removes the original +and keeps the emitted families; an error discards both. + +Scalar values use the OpenTelemetry int64 and double types. Native unsigned +64-bit values up to 2^63 − 1 are passed to the policy as int64. Families with +larger values are not passed to the policy; `error_mode` decides whether they +are kept unchanged or dropped. + +## Traces + +`process(span)` runs once per span. Resource and scope changes apply to that +span only. + +```lua +local fb = require("fluentbit") + +function process(span) + if span.attributes:get_string("http.route") == "/health" then + return fb.DROP + end + + fb.redact(span.attributes, {"authorization", "password"}) + fb.redact(span.events, {"authorization", "password"}) +end +``` + +| Property | Description | +| -------- | ----------- | +| `span.name`, `.trace_state`, `.kind`, `.flags` | Span fields. `kind` is the numeric OpenTelemetry span kind. | +| `span.trace_id`, `.span_id`, `.parent_span_id` | Hexadecimal IDs. An empty parent ID marks a root span. | +| `span.start_time_unix_nano`, `.end_time_unix_nano` | Exact nanosecond timestamps. The end must not precede the start. | +| `span.attributes` | Attribute map. | +| `span.status` | Map with `code` and `message`. | +| `span.events`, `span.links` | Arrays of event and link maps. | +| `span.dropped_attributes_count`, `.dropped_events_count`, `.dropped_links_count` | Dropped counts. | +| `span.resource`, `span.scope` | Same properties as for logs. | + +Dropping spans or changing IDs can break trace topology; FTS does not repair +parent references. + +## Common helpers + +| Helper | Description | +| ------ | ----------- | +| `fb.redact(value, keys, replacement)` | Replace the values of matching keys in nested maps and arrays. Key matching is case-insensitive. The default replacement is `[REDACTED]`. | +| `fb.keep_keys(map, keys)` | Remove every key not listed. | +| `fb.parse_json(text)` | Parse a JSON document into typed values. Integer widths are preserved; duplicate keys and nesting beyond 64 levels are rejected. Input is limited to 1 MiB. | +| `fb.parse_time(text)` | Parse an RFC 3339 timestamp into `{sec, nsec}`. | + +Key-based redaction does not inspect free-form strings; secrets embedded in +message text need an explicit string policy. + +## Limits and sandbox + +- Values can have at most 64 levels of nesting. +- OpenTelemetry strings and keys must be valid UTF-8 without NUL characters; + use `fb.bytes` for binary data. +- Only `require("fluentbit")` is available. The filesystem, process, FFI, + debug, coroutine, and module loading libraries are disabled, as are `print`, + `xpcall`, `getfenv`, `setfenv`, and `module`. `collectgarbage` accepts only + `collect`, `count`, and `step`. +- The instruction limit counts Lua instructions, not elapsed time. Lua string + pattern functions run in C and are not interrupted; avoid unbounded patterns + on untrusted input. +- With `jit: true` there is no CPU limit. Use it only for policies whose + execution time you control. +- The sandbox is not a security boundary: policies are trusted configuration. +- Each processor instance has its own Lua state. + +## Diagnostics + +Each chunk logs at most five policy errors plus a summary count. The processor +error metric counts chunks with handled policy failures, and the drop metrics +count removed records. diff --git a/plugins/processor_fts/fts.c b/plugins/processor_fts/fts.c new file mode 100644 index 00000000000..2393aa8c709 --- /dev/null +++ b/plugins/processor_fts/fts.c @@ -0,0 +1,197 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#include "fts.h" +#include "fts_telemetry.h" + +static int cb_exit(struct flb_processor_instance *ins, void *data) +{ + struct fts_runtime *ctx = data; + + if (!ctx) { + return FLB_PROCESSOR_SUCCESS; + } + + fts_runtime_destroy(ctx); + flb_free(ctx); + + return FLB_PROCESSOR_SUCCESS; +} + +static int config_validate(struct flb_processor_instance *ins, struct fts_runtime *ctx) +{ + if ((!ctx->script && !ctx->code) || (ctx->script && ctx->code)) { + flb_plg_error(ins, "FTS requires exactly one of script or code"); + return -1; + } + + if (strcmp(ctx->error_mode, "keep") != 0 && strcmp(ctx->error_mode, "drop") != 0) { + flb_plg_error(ins, "error_mode must be keep or drop"); + return -1; + } + + if (ctx->jit && ctx->instruction_limit != 0) { + flb_plg_error(ins, "jit requires instruction_limit=0"); + return -1; + } + + if (!ctx->jit && ctx->instruction_limit < 1000) { + flb_plg_error(ins, "instruction_limit must be between 1000 and %d with jit disabled", + INT_MAX); + return -1; + } + + if (ctx->memory_limit < 262144) { + flb_plg_error(ins, "memory_limit must be at least 262144 bytes (256 KiB)"); + return -1; + } + + if (ctx->output_limit < 65536) { + flb_plg_error(ins, "output_limit must be at least 65536 bytes (64 KiB)"); + return -1; + } + + return 0; +} + +static int cb_init(struct flb_processor_instance *ins, void *source, int type, + struct flb_config *config) +{ + int ret; + lua_State *l; + struct fts_runtime *ctx; + + if (ins->event_type != FLB_PROCESSOR_LOGS && ins->event_type != FLB_PROCESSOR_METRICS && + ins->event_type != FLB_PROCESSOR_TRACES) { + flb_plg_error(ins, "FTS supports logs, metrics and traces"); + return FLB_PROCESSOR_FAILURE; + } + + ctx = flb_calloc(1, sizeof(struct fts_runtime)); + if (!ctx) { + flb_errno(); + return FLB_PROCESSOR_FAILURE; + } + + if (ins->event_type == FLB_PROCESSOR_LOGS) { + ctx->adapter = &fts_logs_adapter; + } + else if (ins->event_type == FLB_PROCESSOR_METRICS) { + ctx->adapter = &fts_metrics_adapter; + } + else { + ctx->adapter = &fts_traces_adapter; + } + + ret = flb_processor_instance_config_map_set(ins, ctx); + if (ret != 0) { + flb_plg_error(ins, "invalid FTS configuration value"); + cb_exit(ins, ctx); + return FLB_PROCESSOR_FAILURE; + } + + if (config_validate(ins, ctx) != 0) { + cb_exit(ins, ctx); + return FLB_PROCESSOR_FAILURE; + } + + /* Load the helpers and the policy script into a private Lua state */ + ret = fts_runtime_initialize(ctx, config); + if (ret != FLB_PROCESSOR_SUCCESS) { + l = ctx->vm ? ctx->vm->state : NULL; + if (!l) { + flb_plg_error(ins, "FTS initialization failed: VM allocation error"); + } + else if (lua_type(l, -1) == LUA_TSTRING) { + flb_plg_error(ins, "FTS initialization failed: %.256s", lua_tostring(l, -1)); + } + else { + flb_plg_error(ins, "FTS initialization failed: non-string error"); + } + cb_exit(ins, ctx); + return FLB_PROCESSOR_FAILURE; + } + + flb_processor_instance_set_context(ins, ctx); + + return FLB_PROCESSOR_SUCCESS; +} + +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_STR, "script", NULL, + 0, FLB_TRUE, offsetof(struct fts_runtime, script), + "Path to the Lua policy script. Exclusive with 'code'." + }, + { + FLB_CONFIG_MAP_STR, "code", NULL, + 0, FLB_TRUE, offsetof(struct fts_runtime, code), + "Inline Lua policy script. Exclusive with 'script'." + }, + { + FLB_CONFIG_MAP_STR, "call", "process", + 0, FLB_TRUE, offsetof(struct fts_runtime, call), + "Name of the Lua function invoked for every telemetry item." + }, + { + FLB_CONFIG_MAP_STR, "error_mode", "drop", + 0, FLB_TRUE, offsetof(struct fts_runtime, error_mode), + "Action when a policy fails: 'drop' the item or 'keep' the original." + }, + { + FLB_CONFIG_MAP_BOOL, "jit", "false", + 0, FLB_TRUE, offsetof(struct fts_runtime, jit), + "Enable the LuaJIT compiler for trusted policies. Requires instruction_limit=0." + }, + { + FLB_CONFIG_MAP_INT, "instruction_limit", "1000000", + 0, FLB_TRUE, offsetof(struct fts_runtime, instruction_limit), + "Maximum number of Lua instructions per callback." + }, + { + FLB_CONFIG_MAP_SIZE, "output_limit", "64M", + 0, FLB_TRUE, offsetof(struct fts_runtime, output_limit), + "Maximum serialized size of a processed chunk." + }, + { + FLB_CONFIG_MAP_SIZE, "memory_limit", "32M", + 0, FLB_TRUE, offsetof(struct fts_runtime, memory_limit), + "Maximum memory used by the policy: Lua heap plus typed values." + }, + + /* EOF */ + {0} +}; + +struct flb_processor_plugin processor_fts_plugin = { + .name = "fts", + .description = "Fluent Typed Scripting: Lua-powered typed telemetry policies", + .cb_init = cb_init, + .cb_process_logs = fts_process_logs, + .cb_process_metrics = fts_process_metrics, + .cb_process_traces = fts_process_traces, + .cb_exit = cb_exit, + .config_map = config_map, + .flags = 0 +}; diff --git a/plugins/processor_fts/fts.h b/plugins/processor_fts/fts.h new file mode 100644 index 00000000000..f3112ec4e1d --- /dev/null +++ b/plugins/processor_fts/fts.h @@ -0,0 +1,153 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PROCESSOR_FTS_H +#define FLB_PROCESSOR_FTS_H + +#include +#include +#include + +#define FTS_MAX_DEPTH 64 + +/* Each signal owns its representation, validation and commit semantics. */ +struct fts_signal_adapter { + int event_type; + int (*register_helpers)(lua_State *l); + lua_CFunction run_callback; +}; + +enum fts_key { + FTS_KEY_TAG, FTS_KEY_BODY, FTS_KEY_METADATA, FTS_KEY_TIMESTAMP, + FTS_KEY_SEC, FTS_KEY_NSEC, FTS_KEY_CONTEXT, FTS_KEY_ATTRIBUTES, + FTS_KEY_COUNT +}; + +struct fts_runtime; + +/* Reversible in-place edits of chunk-owned log values. */ +struct fts_undo_entry { + int type; + struct cfl_list *list; + struct cfl_kvpair *pair; + struct cfl_list *prev; + struct cfl_array *array; + size_t index; + struct cfl_variant *old; +}; + +struct fts_value { + struct cfl_variant *owned; + unsigned int readonly : 1; + unsigned int dirty : 1; + unsigned int depth_changed : 1; + unsigned int borrowed : 1; + unsigned int record : 1; + unsigned int inplace : 1; + struct fts_runtime *ctx; + uint64_t generation; + size_t charged; + + /* + * Views: owning root and the node resolved at the root's last epoch. + * Roots: epoch advances on every write, invalidating view caches. + */ + struct fts_value *root; + struct cfl_variant *cache; + uint64_t epoch; +}; + +struct fts_runtime { + struct flb_luajit *vm; + char *script; + char *code; + char *call; + char *error_mode; + const struct fts_signal_adapter *adapter; + const void *value_metatable; + int value_metatable_ref; + const void *empty_metatable; + int empty_metatable_ref; + int key_refs[FTS_KEY_COUNT]; + int tag_ref; + uint64_t generation; + struct fts_undo_entry *undo; + size_t undo_count; + size_t undo_size; + size_t undo_held; + size_t undo_charged; + int inplace_ok; + int tag_ready; + int callback_ref; + int policy_ref; + int module_ref; + int helpers_env_ref; + int jit; + int instruction_limit; + int instructions_left; + size_t memory_limit; + size_t lua_bytes; + size_t native_bytes; + size_t native_allocated; /* native bytes charged since the last GC pacing */ + size_t output_limit; + lua_Alloc original_allocator; + void *allocator_data; +}; + +extern const struct fts_signal_adapter fts_logs_adapter; + +/* runtime: Lua VM lifecycle, policy calls and error reporting */ +int fts_runtime_initialize(struct fts_runtime *ctx, struct flb_config *config); +void fts_runtime_destroy(struct fts_runtime *ctx); +void fts_report_policy_error(struct flb_processor_instance *ins); +int fts_runtime_call(struct fts_runtime *ctx, void *data, int results); +int fts_register_values(lua_State *l); +void fts_collect(struct fts_runtime *ctx, int emergency); + +/* cached field-name keys */ +void fts_getfield(lua_State *l, int index, enum fts_key key); +void fts_setfield(lua_State *l, int index, enum fts_key key); + +/* Lua values backed by msgpack or cfl_variant data */ +struct fts_value *fts_push_empty_map(lua_State *l); +struct fts_value *fts_push_borrowed(lua_State *l, struct cfl_variant *source); + +/* undo log for in-place edits */ +void fts_undo_rollback(struct fts_runtime *ctx, size_t mark); +void fts_undo_commit(struct fts_runtime *ctx); + +/* helpers and value utilities */ +int fts_load_helpers(lua_State *l, const char *code, size_t size, const char *name); +int fts_values_equal(struct cfl_variant *a, struct cfl_variant *b, int depth); +int fts_valid_utf8(const char *text, size_t length); +struct cfl_variant *fts_copy_value(struct cfl_variant *value, int depth); +struct cfl_variant *fts_copy_value_in(struct cfl_variant *value, int depth, + struct cfl_arena *arena); +size_t fts_value_cost(struct cfl_variant *value, int depth); +void fts_charge_value(lua_State *l, struct fts_value *owner, size_t bytes); +struct cfl_kvpair *fts_find_pair(struct cfl_variant *value, const char *key, size_t length); +struct cfl_variant *fts_resolve_value(lua_State *l, int index, int depth); +struct fts_value *fts_push_owned(lua_State *l, struct cfl_variant *value, int readonly); + +/* logs signal */ + +int fts_process_logs(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len); + +#endif diff --git a/plugins/processor_fts/fts_helpers.h.in b/plugins/processor_fts/fts_helpers.h.in new file mode 100644 index 00000000000..43c7d4afdff --- /dev/null +++ b/plugins/processor_fts/fts_helpers.h.in @@ -0,0 +1,15 @@ +/* Generated from the processor_fts Lua helper sources. Do not edit. */ + +#ifndef FLB_PROCESSOR_FTS_HELPERS_H +#define FLB_PROCESSOR_FTS_HELPERS_H + +static const char fts_value_helpers[] = +"@FTS_VALUE_HELPERS@"; + +static const char fts_log_helpers[] = +"@FTS_LOG_HELPERS@"; + +static const char fts_telemetry_helpers[] = +"@FTS_TELEMETRY_HELPERS@"; + +#endif diff --git a/plugins/processor_fts/fts_log_helpers.lua b/plugins/processor_fts/fts_log_helpers.lua new file mode 100644 index 00000000000..f19f9d7a39c --- /dev/null +++ b/plugins/processor_fts/fts_log_helpers.lua @@ -0,0 +1,171 @@ +local fb = package.loaded.fluentbit +function fb.is_otel(log) + local schema = rawget(log, 'context') and rawget(log, 'context').metadata:get('schema') + return schema and schema:type() == 'string' and schema:value() == 'otlp' or false +end +local function read_map(map, key, created) + local value = map:get(key) + if value then return value end + value = map:ensure_map(key) + created[#created+1] = {map, key} + return value +end +local function semantic(map, created) + return setmetatable({}, { + __index = function(_, key) + if key == 'attributes' then return read_map(map, key, created) end + local value = map:get(key); if value then return value:value() end + end, + __newindex = function(_, key, value) + assert(key == 'name' or key == 'version' or key == 'schema_url', 'unsupported context property') + assert(type(value) == 'string', 'context property must be a string') + map:set(key, value) + end, + __metatable = 'telemetry context' + }) +end +function fb._attach(log) + local created + local context = rawget(log, 'context') + local root_metadata = rawget(log, 'metadata') + local otel = fb.is_otel(log) + local function identity(name) + -- Missing IDs are regenerated when the group is rebuilt. + local value = rawget(context, 'metadata'):get(name) + if value == nil then return nil end + assert(value:type() == 'int64' or value:type() == 'uint64', 'invalid transport ID') + return value:value() + end + local resource_id, scope_id + if otel then + created = {} + resource_id = identity('resource_id') + scope_id = identity('scope_id') + log.resource = semantic(read_map(rawget(context, 'attributes'), 'resource', created), created) + log.scope = semantic(read_map(rawget(context, 'attributes'), 'scope', created), created) + log.attributes = read_map(read_map(rawget(log, 'metadata'), 'otlp', created), 'attributes', created) + end + local resource, scope, attrs = rawget(log, 'resource'), rawget(log, 'scope'), rawget(log, 'attributes') + local original_time = otel and fb.log_field(log, 'timestamp') + original_time = original_time and original_time:value() + local function validate(sec, nsec) + assert(rawget(log, 'context') == context, 'context root cannot be replaced') + assert(rawget(log, 'metadata') == root_metadata, 'metadata root cannot be replaced') + assert(fb.is_otel(log) == otel, 'transport schema cannot be changed') + if otel then + assert(rawget(log, 'resource') == resource and rawget(log, 'scope') == scope and rawget(log, 'attributes') == attrs, + 'semantic aliases cannot be replaced') + assert(rawget(context, 'metadata'):get_string('schema') == 'otlp' + and identity('resource_id') == resource_id + and identity('scope_id') == scope_id, + 'transport identifiers cannot be changed') + if rawget(log, 'timestamp').sec == sec and rawget(log, 'timestamp').nsec == nsec then + local current = fb.log_field(log, 'timestamp') + assert((current and current:value()) == original_time, 'change event time through log.timestamp') + end + end + end + return validate,created +end +function fb._cleanup(created) + for i=#created,1,-1 do + local map,key = created[i][1],created[i][2] + -- A policy may have removed an ancestor through the advanced context API. + local ok,value = pcall(function() return map:get(key) end) + if ok and value and value:type() == 'map' and value:len() == 0 then + map:remove(key) + end + end +end +function fb.attributes(log) + assert(fb.is_otel(log), 'OTel log required') + return rawget(log, 'attributes') +end +function fb.log_field(log, name) + assert(fb.is_otel(log), 'OTel log required') + local otlp = rawget(log, 'metadata'):get('otlp') + return (otlp and otlp:get(name)) or rawget(log, 'metadata'):get(name) +end +function fb.set_severity(log, level) + assert(fb.is_otel(log), 'OTel log required') + local names = {trace=1, debug=5, info=9, information=9, informational=9, + warn=13, warning=13, error=17, fatal=21, critical=21} + assert(type(level) == 'string', 'severity must be a named level') + local number = names[level:lower()]; assert(number, 'unknown severity level') + local otlp = rawget(log, 'metadata'):ensure_map('otlp') + otlp:set('severity_text', level); otlp:set('severity_number', fb.int64(number)) +end +function fb.set_trace_context(log, trace, span) + assert(fb.is_otel(log), 'OTel log required') + local function decode(s, length) + assert(type(s) == 'string' and #s == length * 2 and not s:find('[^%x]') + and s:find('[1-9a-fA-F]'), 'invalid correlation ID') + return fb.bytes((s:gsub('..', function(pair) return string.char(tonumber(pair, 16)) end))) + end + local trace_value = decode(trace, 16); local span_value = decode(span, 8) + local otlp = rawget(log, 'metadata'):ensure_map('otlp') + otlp:set('trace_id', trace_value); otlp:set('span_id', span_value) +end +function fb._validate(log, original_sec, original_nsec, changed) + if not fb.is_otel(log) then return end + for _,kind in ipairs({'resource','scope'}) do + local map = rawget(log, 'context').attributes:get(kind) + if map then + assert(map:type() == 'map', kind .. ' must be a map') + for _,name in ipairs({'schema_url','name','version'}) do + local value = map:get(name) + assert(not changed[kind .. '.' .. name] or not value or value:type() == 'string', kind .. '.' .. name .. ' must be a string') + end + local attrs = map:get('attributes') + if attrs and changed[kind .. '.attributes'] then + assert(attrs:type() == 'map', kind .. '.attributes must be a map') + local seen = {} + for _,key in ipairs(attrs:keys()) do + assert(not seen[key], 'duplicate attribute key'); seen[key] = true + end + end + local count = map:get('dropped_attributes_count') + if count and changed[kind .. '.dropped_attributes_count'] then + assert(count:type() == 'int64' or count:type() == 'uint64', 'dropped count must be an integer') + assert(count:number() >= 0 and count:number() <= 4294967295, 'dropped count out of range') + end + end + end + assert(rawget(log, 'attributes'):type() == 'map', 'log.attributes must be a map') + local seen = {} + for _,key in ipairs(changed.attributes and rawget(log, 'attributes'):keys() or {}) do + assert(not seen[key], 'duplicate attribute key'); seen[key] = true + end + local otlp = rawget(log, 'metadata'):get('otlp') + for _,name in ipairs({'severity_number','trace_flags','dropped_attributes_count'}) do + local value = fb.log_field(log, name) + if value and changed[name] then + assert(value:type() == 'int64' or value:type() == 'uint64', name .. ' must be an integer') + local number = value:number() + assert(number>=0 and number <= (name == 'severity_number' and 24 or (name == 'trace_flags' and 255 or 4294967295)), name .. ' out of range') + end + end + for _,name in ipairs({'trace_id','span_id'}) do + local value = fb.log_field(log, name) + if value and changed[name] then + local str = value:value() + assert(value:type() == 'bytes' and #str == (name == 'trace_id' and 16 or 8) + and not str:match('^%z+$'), 'invalid ' .. name) + end + end + local name = fb.log_field(log, 'event_name') + assert(not changed.event_name or not name or name:type() == 'string', 'event_name must be a string') + local severity = fb.log_field(log, 'severity_text') + assert(not changed.severity_text or not severity or severity:type() == 'string', 'severity_text must be a string') + for _,name in ipairs({'timestamp','observed_timestamp'}) do + local value = fb.log_field(log, name) + if value and changed[name] then + assert(value:type() == 'uint64' or (value:type() == 'int64' and value:value():sub(1,1) ~= '-'), + name .. ' must be a nonnegative integer') + end + end + if rawget(log, 'timestamp').sec ~= original_sec or rawget(log, 'timestamp').nsec ~= original_nsec then + local str = string.format('%.0f', rawget(log, 'timestamp').sec) .. string.format('%09d',rawget(log, 'timestamp').nsec) + otlp:set('timestamp', fb.uint64(str)) + end +end diff --git a/plugins/processor_fts/fts_logs.c b/plugins/processor_fts/fts_logs.c new file mode 100644 index 00000000000..93dc0716944 --- /dev/null +++ b/plugins/processor_fts/fts_logs.c @@ -0,0 +1,1569 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "fts.h" +#include "fts_helpers.h" + +/* Log semantics remain in the log adapter. */ +static int values_equal(struct cfl_variant *a, struct cfl_variant *b); +static struct cfl_object *object_from_variant(struct cfl_variant *v); + +/* All Lua allocation, callbacks and validation run within one protected call. */ +struct record_call { + struct fts_runtime *ctx; + struct flb_mp_chunk_cobj *chunk; + struct cfl_variant *result_body; + struct cfl_variant *result_metadata; + struct flb_mp_chunk_record *record; + struct cfl_object *attributes; + struct cfl_object *metadata; + const char *tag; + int tag_len; +}; + +static struct fts_value *push_clone_in(lua_State *l, struct cfl_variant *source, + struct flb_mp_chunk_cobj *chunk) +{ + struct fts_value *value; + struct cfl_arena *arena; + size_t cost; + + value = fts_push_owned(l, NULL, FLB_FALSE); + cost = fts_value_cost(source, 0); + fts_charge_value(l, value, cost); + + arena = flb_mp_chunk_cobj_value_pool_get(chunk, cost > SIZE_MAX / 4 ? SIZE_MAX : cost * 4); + value->owned = fts_copy_value_in(source, 0, arena); + if (!value->owned && arena) { + value->owned = fts_copy_value(source, 0); + } + + if (!value->owned) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + + return value; +} + +static struct fts_value *push_clone(lua_State *l, struct cfl_variant *source) +{ + return push_clone_in(l, source, NULL); +} + +static int has_otel_context(struct cfl_object *obj) +{ + struct cfl_kvpair *pair; + + if (!obj || !obj->variant || obj->variant->type != CFL_VARIANT_KVLIST) { + return FLB_FALSE; + } + + pair = fts_find_pair(obj->variant, "schema", 6); + return pair && pair->val->type == CFL_VARIANT_STRING && + cfl_variant_size_get(pair->val) == 4 && + memcmp(pair->val->data.as_string, "otlp", 4) == 0; +} + +static struct cfl_variant *raw_field(lua_State *l, int table, enum fts_key key) +{ + struct cfl_variant *v; + + fts_getfield(l, table, key); + v = fts_resolve_value(l, -1, 0); + lua_pop(l, 1); + + return v; +} + +/* + * Push the field to pin its owner; return its tree only when the policy changed it. + * The caller copies it into the chunk after the protected call. + */ +static struct cfl_variant *push_record_result(lua_State *l, int table, enum fts_key key, + struct fts_value *original) +{ + struct cfl_variant *value; + + fts_getfield(l, table, key); + if (lua_touserdata(l, -1) == original && (!original->dirty || original->inplace)) { + /* In-place edits already live in the chunk's tree. */ + if (original->inplace && original->depth_changed && + fts_value_cost(original->owned, 0) == SIZE_MAX) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + return NULL; + } + + value = fts_resolve_value(l, -1, 0); + + /* Keep nesting failures item-local instead of failing the chunk copy. */ + if ((lua_touserdata(l, -1) != original || original->depth_changed) && + fts_value_cost(value, 0) == SIZE_MAX) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + + return value; +} + +/* Lua may retain policy trees, so the chunk receives its own copy. */ +static struct cfl_object *result_object(struct flb_mp_chunk_cobj *chunk, + struct cfl_variant *source) +{ + struct cfl_variant *value; + struct cfl_object *obj; + struct cfl_arena *arena; + size_t cost; + + cost = fts_value_cost(source, 0); + if (cost == SIZE_MAX) { + return NULL; + } + + arena = flb_mp_chunk_cobj_value_pool_get(chunk, cost > SIZE_MAX / 4 ? SIZE_MAX : cost * 4); + value = fts_copy_value_in(source, 0, arena); + if (!value && arena) { + value = fts_copy_value(source, 0); + } + + if (!value) { + return NULL; + } + + obj = object_from_variant(value); + if (!obj) { + cfl_variant_destroy(value); + } + + return obj; +} + +static void push_field_clone(lua_State *l, int table, enum fts_key key) +{ + struct cfl_variant *v; + + v = raw_field(l, table, key); + push_clone(l, v); +} + +static void validate_map(lua_State *l, struct cfl_variant *v, const char *name) +{ + if (v == NULL || v->type != CFL_VARIANT_KVLIST) { + luaL_error(l, "%s must be a typed map", name); + } +} + +static void validate_utf8(lua_State *l, struct cfl_variant *v, struct cfl_variant *original, + int depth, int payload) +{ + struct cfl_list *head; + struct cfl_list *original_head; + struct cfl_list *original_end; + struct cfl_kvpair *pair; + struct cfl_kvpair *previous; + struct cfl_variant *old_value; + size_t i; + size_t original_index = 0; + size_t original_count; + int same_key; + + if (values_equal(v, original) || v == NULL) { + return; + } + + if (depth >= FTS_MAX_DEPTH) { + luaL_error(l, "telemetry nesting exceeds limit"); + } + + if (payload && v->type == CFL_VARIANT_UINT && v->data.as_uint64 > INT64_MAX) { + luaL_error(l, "OTel AnyValue integers must fit signed 64 bits"); + } + + if (v->type == CFL_VARIANT_STRING && + memchr(v->data.as_string, '\0', cfl_variant_size_get(v))) { + luaL_error(l, "OTel strings cannot contain NUL in this exporter; use fb.bytes"); + } + + if (v->type == CFL_VARIANT_STRING && + !fts_valid_utf8(v->data.as_string, cfl_variant_size_get(v))) { + luaL_error(l, "OTel strings must be valid UTF-8; use fb.bytes for binary values"); + } + else if (v->type == CFL_VARIANT_KVLIST) { + original_end = original && original->type == CFL_VARIANT_KVLIST ? + &original->data.as_kvlist->list : NULL; + original_head = original_end ? original_end->next : NULL; + + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + previous = NULL; + + /* Set/remove preserve order; match duplicate occurrences positionally. */ + while (original_head != original_end) { + previous = cfl_list_entry(original_head, struct cfl_kvpair, _head); + original_head = original_head->next; + same_key = cfl_sds_len(previous->key) == cfl_sds_len(pair->key) && + memcmp(previous->key, pair->key, cfl_sds_len(pair->key)) == 0; + if (same_key) { + break; + } + previous = NULL; + } + + /* Only new keys need validation; inherited keys were already accepted. */ + if (previous == NULL) { + if (memchr(pair->key, '\0', cfl_sds_len(pair->key))) { + luaL_error(l, "OTel keys cannot contain NUL in this exporter"); + } + if (!fts_valid_utf8(pair->key, cfl_sds_len(pair->key))) { + luaL_error(l, "OTel keys must be valid UTF-8"); + } + } + + validate_utf8(l, pair->val, previous ? previous->val : NULL, depth + 1, payload); + } + } + else if (v->type == CFL_VARIANT_ARRAY) { + original_count = original && original->type == CFL_VARIANT_ARRAY ? + original->data.as_array->entry_count : 0; + + for (i = 0; i < v->data.as_array->entry_count; i++) { + old_value = original_index < original_count ? + original->data.as_array->entries[original_index] : NULL; + + if (old_value && !values_equal(v->data.as_array->entries[i], old_value) && + v->data.as_array->entry_count - i > original_count - original_index) { + /* An inserted value must not shift validation of inherited values. */ + old_value = NULL; + } + else if (old_value) { + original_index++; + } + + validate_utf8(l, v->data.as_array->entries[i], old_value, depth + 1, payload); + } + } +} + +static struct cfl_variant *map_field(struct cfl_variant *map, const char *key) +{ + struct cfl_kvpair *pair; + + if (!map || map->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + pair = fts_find_pair(map, key, strlen(key)); + return pair ? pair->val : NULL; +} + +static struct cfl_variant *log_field(struct cfl_variant *metadata, const char *key) +{ + struct cfl_variant *value; + + value = map_field(map_field(metadata, "otlp"), key); + return value ? value : map_field(metadata, key); +} + +static void push_changed_fields(lua_State *l, struct record_call *call, + struct cfl_variant *metadata, struct cfl_variant *context) +{ + const char *fields[] = {"severity_number", "severity_text", "trace_flags", "trace_id", + "span_id", "event_name", "timestamp", "observed_timestamp", + "attributes", "dropped_attributes_count", NULL}; + const char *context_keys[] = {"resource", "scope", NULL}; + const char *properties[] = {"name", "version", "schema_url", "attributes", + "dropped_attributes_count", NULL}; + struct cfl_variant *current; + struct cfl_variant *original; + char key[64]; + int i; + int j; + + lua_newtable(l); + + /* Log record fields */ + for (i = 0; fields[i]; i++) { + lua_pushboolean(l, !values_equal(log_field(metadata, fields[i]), + log_field(call->record->cobj_metadata->variant, + fields[i]))); + lua_setfield(l, -2, fields[i]); + } + + /* Resource and scope properties, keyed as 'kind.property' */ + for (i = 0; context_keys[i]; i++) { + current = map_field(context, context_keys[i]); + original = map_field(call->attributes->variant, context_keys[i]); + + for (j = 0; properties[j]; j++) { + snprintf(key, sizeof(key), "%s.%s", context_keys[i], properties[j]); + lua_pushboolean(l, !values_equal(map_field(current, properties[j]), + map_field(original, properties[j]))); + lua_setfield(l, -2, key); + } + } +} + +static int empty_context_map(struct cfl_variant *value, int kind) +{ + struct cfl_kvpair *pair; + + if (!value || value->type != CFL_VARIANT_KVLIST) { + return value == NULL; + } + + if (cfl_kvlist_count(value->data.as_kvlist) == 0) { + return FLB_TRUE; + } + + if (kind && cfl_kvlist_count(value->data.as_kvlist) == 1) { + pair = fts_find_pair(value, "attributes", 10); + return pair && empty_context_map(pair->val, FLB_FALSE); + } + + return FLB_FALSE; +} + +static int context_kind_equal(struct cfl_variant *left, struct cfl_variant *right) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_variant *other; + + if (!left || !right) { + return empty_context_map(left, FLB_TRUE) && empty_context_map(right, FLB_TRUE); + } + + if (left->type != CFL_VARIANT_KVLIST || right->type != CFL_VARIANT_KVLIST) { + return values_equal(left, right); + } + + cfl_list_foreach(head, &left->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + other = map_field(right, pair->key); + if (strcmp(pair->key, "attributes") == 0 && + empty_context_map(pair->val, FLB_FALSE) && empty_context_map(other, FLB_FALSE)) { + continue; + } + if (!values_equal(pair->val, other)) { + return FLB_FALSE; + } + } + + cfl_list_foreach(head, &right->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + other = map_field(left, pair->key); + if (!other && + !(strcmp(pair->key, "attributes") == 0 && empty_context_map(pair->val, FLB_FALSE))) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + +static int context_attributes_equal(struct cfl_variant *left, struct cfl_variant *right) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_variant *other; + + if (!left || !right || + left->type != CFL_VARIANT_KVLIST || right->type != CFL_VARIANT_KVLIST) { + return values_equal(left, right); + } + + cfl_list_foreach(head, &left->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + other = map_field(right, pair->key); + if (strcmp(pair->key, "resource") == 0 || strcmp(pair->key, "scope") == 0) { + if (!context_kind_equal(pair->val, other)) { + return FLB_FALSE; + } + } + else if (!values_equal(pair->val, other)) { + return FLB_FALSE; + } + } + + cfl_list_foreach(head, &right->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (!map_field(left, pair->key)) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + +static void require_plain_table(lua_State *l, int index, const char *name) +{ + luaL_checktype(l, index, LUA_TTABLE); + if (lua_getmetatable(l, index)) { + lua_pop(l, 1); + luaL_error(l, "%s must not have a metatable", name); + } +} + +static int run_record(lua_State *l) +{ + struct record_call *call; + struct fts_value *original_body; + struct fts_value *original_metadata; + struct fts_value *attributes = NULL; + struct fts_value *metadata = NULL; + struct cfl_object current_metadata; + const char *result; + size_t result_length; + lua_Number sec; + lua_Number nsec; + int event; + int context = 0; + int validator = 0; + int cleanup = 0; + + call = lua_touserdata(l, 1); + + /* Expire record handles from earlier callbacks before creating new ones. */ + call->ctx->generation++; + + /* Build the event table: tag, body, metadata and timestamp */ + lua_createtable(l, 0, 4); + event = lua_gettop(l); + + if (!call->ctx->tag_ready) { + lua_pushlstring(l, call->tag, call->tag_len); + lua_rawseti(l, LUA_REGISTRYINDEX, call->ctx->tag_ref); + call->ctx->tag_ready = FLB_TRUE; + } + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->tag_ref); + fts_setfield(l, event, FTS_KEY_TAG); + + original_body = fts_push_borrowed(l, call->record->cobj_record->variant); + lua_pushvalue(l, -1); + fts_setfield(l, event, FTS_KEY_BODY); + + original_metadata = fts_push_borrowed(l, call->record->cobj_metadata->variant); + lua_pushvalue(l, -1); + fts_setfield(l, event, FTS_KEY_METADATA); + + lua_createtable(l, 0, 2); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_sec); + fts_setfield(l, -2, FTS_KEY_SEC); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_nsec); + fts_setfield(l, -2, FTS_KEY_NSEC); + fts_setfield(l, event, FTS_KEY_TIMESTAMP); + + /* Group context: attributes and metadata are policy-owned clones */ + if (call->attributes) { + lua_createtable(l, 0, 2); + context = lua_gettop(l); + + attributes = push_clone(l, call->attributes->variant); + + /* Pin C-held userdata independently of mutable table fields. */ + lua_pushvalue(l, -1); + fts_setfield(l, context, FTS_KEY_ATTRIBUTES); + + metadata = push_clone(l, call->metadata->variant); + lua_pushvalue(l, -1); + fts_setfield(l, context, FTS_KEY_METADATA); + + lua_pushvalue(l, context); + fts_setfield(l, event, FTS_KEY_CONTEXT); + + /* Semantic aliases are populated by the module without copying data. */ + } + + if (has_otel_context(call->metadata)) { + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->module_ref); + lua_getfield(l, -1, "_attach"); + lua_pushvalue(l, event); + lua_call(l, 1, 2); + cleanup = lua_gettop(l); + validator = cleanup - 1; + } + + original_metadata->dirty = FLB_FALSE; + if (attributes) { + attributes->dirty = FLB_FALSE; + metadata->dirty = FLB_FALSE; + } + + /* Run the policy callback */ + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->policy_ref); + lua_pushvalue(l, event); + lua_call(l, 1, 1); + + if (!lua_isnil(l, -1)) { + result = lua_type(l, -1) == LUA_TSTRING ? lua_tolstring(l, -1, &result_length) : NULL; + if (result && result_length == 4 && memcmp(result, "drop", 4) == 0) { + lua_pushliteral(l, "drop"); + return 1; + } + if (!result || result_length != 4 || memcmp(result, "keep", 4) != 0) { + return luaL_error(l, "callback must return nil, fb.KEEP or fb.DROP"); + } + } + + /* Validation must not execute policy code through table metamethods. */ + require_plain_table(l, event, "log event"); + + /* Roots: metadata and context cannot be replaced */ + fts_getfield(l, event, FTS_KEY_METADATA); + if (lua_touserdata(l, -1) != original_metadata) { + return luaL_error(l, "metadata root cannot be replaced"); + } + lua_pop(l, 1); + + fts_getfield(l, event, FTS_KEY_CONTEXT); + if (context ? !lua_rawequal(l, -1, context) : !lua_isnil(l, -1)) { + return luaL_error(l, "context root cannot be replaced"); + } + lua_pop(l, 1); + + if (context) { + require_plain_table(l, context, "log context"); + } + + validate_map(l, raw_field(l, event, FTS_KEY_BODY), "body"); + validate_map(l, raw_field(l, event, FTS_KEY_METADATA), "metadata"); + + /* Timestamp: validate only when the policy changed it */ + fts_getfield(l, event, FTS_KEY_TIMESTAMP); + require_plain_table(l, -1, "log timestamp"); + fts_getfield(l, -1, FTS_KEY_SEC); + fts_getfield(l, -2, FTS_KEY_NSEC); + if (lua_type(l, -2) != LUA_TNUMBER || lua_type(l, -1) != LUA_TNUMBER) { + return luaL_error(l, "timestamp sec/nsec must be numbers"); + } + + sec = lua_tonumber(l, -2); + nsec = lua_tonumber(l, -1); + if ((sec != (lua_Number) call->record->event.timestamp.tm.tv_sec || + nsec != (lua_Number) call->record->event.timestamp.tm.tv_nsec) && + (!isfinite(sec) || sec < 0 || + sec > (sizeof(time_t) > 4 ? 4294967293.0 : 2147483647.0) || + sec != floor(sec) || !isfinite(nsec) || nsec < 0 || nsec > 999999999 || + nsec != floor(nsec))) { + return luaL_error(l, "timestamp out of range"); + } + lua_pop(l, 3); + + if (validator) { + lua_pushvalue(l, validator); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_sec); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_nsec); + lua_call(l, 2, 0); + } + + /* Context fields may only change through set/remove */ + if (context) { + if (raw_field(l, context, FTS_KEY_ATTRIBUTES) != attributes->owned || + raw_field(l, context, FTS_KEY_METADATA) != metadata->owned) { + return luaL_error(l, "replace context fields through set/remove, not root assignment"); + } + + validate_map(l, raw_field(l, context, FTS_KEY_ATTRIBUTES), "context attributes"); + validate_map(l, raw_field(l, context, FTS_KEY_METADATA), "context metadata"); + + if (!validator) { + memset(¤t_metadata, 0, sizeof(current_metadata)); + current_metadata.variant = metadata->owned; + if (has_otel_context(¤t_metadata)) { + return luaL_error(l, "transport schema cannot be changed"); + } + } + } + + /* OTLP semantic validation */ + if (has_otel_context(call->metadata)) { + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->module_ref); + lua_getfield(l, -1, "_validate"); + lua_pushvalue(l, event); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_sec); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_nsec); + push_changed_fields(l, call, raw_field(l, event, FTS_KEY_METADATA), attributes->owned); + lua_call(l, 4, 0); + + validate_utf8(l, raw_field(l, event, FTS_KEY_BODY), + call->record->cobj_record->variant, 0, FLB_TRUE); + validate_utf8(l, raw_field(l, event, FTS_KEY_METADATA), + call->record->cobj_metadata->variant, 0, FLB_FALSE); + validate_utf8(l, log_field(raw_field(l, event, FTS_KEY_METADATA), "attributes"), + log_field(call->record->cobj_metadata->variant, "attributes"), + 0, FLB_TRUE); + validate_utf8(l, attributes->owned, call->attributes->variant, 0, FLB_TRUE); + validate_utf8(l, metadata->owned, call->metadata->variant, 0, FLB_FALSE); + } + + if (cleanup && lua_istable(l, cleanup) && + (original_metadata->dirty || (attributes && attributes->dirty))) { + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->module_ref); + lua_getfield(l, -1, "_cleanup"); + lua_pushvalue(l, cleanup); + lua_call(l, 1, 0); + lua_pop(l, 1); + } + + /* + * Determine final changes only after all helpers have finished. Results: + * "keep", body, metadata, context attributes, context metadata, sec, nsec. + */ + lua_pushliteral(l, "keep"); + + call->result_body = push_record_result(l, event, FTS_KEY_BODY, original_body); + call->result_metadata = push_record_result(l, event, FTS_KEY_METADATA, original_metadata); + + if (context && attributes->dirty && + !(has_otel_context(call->metadata) ? + context_attributes_equal(attributes->owned, call->attributes->variant) : + values_equal(attributes->owned, call->attributes->variant))) { + push_field_clone(l, context, FTS_KEY_ATTRIBUTES); + } + else { + lua_pushnil(l); + } + + if (context && metadata->dirty && !values_equal(metadata->owned, call->metadata->variant)) { + push_field_clone(l, context, FTS_KEY_METADATA); + } + else { + lua_pushnil(l); + } + + lua_pushnumber(l, sec); + lua_pushnumber(l, nsec); + + return 7; +} + +static struct cfl_object *object_from_variant(struct cfl_variant *v) +{ + struct cfl_object *obj; + + obj = cfl_object_create(); + if (!obj) { + return NULL; + } + + obj->type = CFL_OBJECT_KVLIST; + obj->variant = v; + + return obj; +} + +static struct cfl_object *copy_object(struct cfl_object *source) +{ + struct cfl_variant *v; + struct cfl_object *obj; + + v = fts_copy_value(source->variant, 0); + if (!v) { + return NULL; + } + + obj = object_from_variant(v); + if (!obj) { + cfl_variant_destroy(v); + } + + return obj; +} + +static int values_equal(struct cfl_variant *a, struct cfl_variant *b) +{ + struct cfl_list *left; + struct cfl_list *right; + struct cfl_kvpair *pa; + struct cfl_kvpair *pb; + size_t i; + + if (!a || !b || a == b) { + return a == b; + } + + if (a->type != b->type) { + return FLB_FALSE; + } + + switch (a->type) { + case CFL_VARIANT_NULL: + return FLB_TRUE; + case CFL_VARIANT_BOOL: + return a->data.as_bool == b->data.as_bool; + case CFL_VARIANT_INT: + return a->data.as_int64 == b->data.as_int64; + case CFL_VARIANT_UINT: + return a->data.as_uint64 == b->data.as_uint64; + case CFL_VARIANT_DOUBLE: + return memcmp(&a->data.as_double, &b->data.as_double, sizeof(double)) == 0; + case CFL_VARIANT_STRING: + case CFL_VARIANT_BYTES: + return cfl_variant_size_get(a) == cfl_variant_size_get(b) && + memcmp(a->data.as_string, b->data.as_string, cfl_variant_size_get(a)) == 0; + case CFL_VARIANT_ARRAY: + if (a->data.as_array->entry_count != b->data.as_array->entry_count) { + return FLB_FALSE; + } + for (i = 0; i < a->data.as_array->entry_count; i++) { + if (!values_equal(a->data.as_array->entries[i], b->data.as_array->entries[i])) { + return FLB_FALSE; + } + } + return FLB_TRUE; + case CFL_VARIANT_KVLIST: + left = a->data.as_kvlist->list.next; + right = b->data.as_kvlist->list.next; + while (left != &a->data.as_kvlist->list && right != &b->data.as_kvlist->list) { + pa = cfl_list_entry(left, struct cfl_kvpair, _head); + pb = cfl_list_entry(right, struct cfl_kvpair, _head); + if (cfl_sds_len(pa->key) != cfl_sds_len(pb->key) || + memcmp(pa->key, pb->key, cfl_sds_len(pa->key)) != 0 || + !values_equal(pa->val, pb->val)) { + return FLB_FALSE; + } + left = left->next; + right = right->next; + } + return left == &a->data.as_kvlist->list && right == &b->data.as_kvlist->list; + default: + return FLB_FALSE; + } +} + +static int context_identity_key(struct cfl_kvpair *pair) +{ + return (cfl_sds_len(pair->key) == 11 && memcmp(pair->key, "resource_id", 11) == 0) || + (cfl_sds_len(pair->key) == 8 && memcmp(pair->key, "scope_id", 8) == 0); +} + +/* Compare group metadata ignoring the transport identity keys. */ +static int context_metadata_equal(struct cfl_variant *left, struct cfl_variant *right) +{ + struct cfl_list *a; + struct cfl_list *b; + struct cfl_kvpair *x; + struct cfl_kvpair *y; + + if (left->type != CFL_VARIANT_KVLIST || right->type != CFL_VARIANT_KVLIST) { + return FLB_FALSE; + } + + a = left->data.as_kvlist->list.next; + b = right->data.as_kvlist->list.next; + + for (;;) { + while (a != &left->data.as_kvlist->list && + context_identity_key(cfl_list_entry(a, struct cfl_kvpair, _head))) { + a = a->next; + } + while (b != &right->data.as_kvlist->list && + context_identity_key(cfl_list_entry(b, struct cfl_kvpair, _head))) { + b = b->next; + } + + if (a == &left->data.as_kvlist->list || b == &right->data.as_kvlist->list) { + return a == &left->data.as_kvlist->list && b == &right->data.as_kvlist->list; + } + + x = cfl_list_entry(a, struct cfl_kvpair, _head); + y = cfl_list_entry(b, struct cfl_kvpair, _head); + if (cfl_sds_len(x->key) != cfl_sds_len(y->key) || + memcmp(x->key, y->key, cfl_sds_len(x->key)) != 0 || + !values_equal(x->val, y->val)) { + return FLB_FALSE; + } + + a = a->next; + b = b->next; + } +} + +/* Replace or add a regenerated transport ID; takes ownership of 'value'. */ +static int set_transport_id(struct cfl_variant *metadata, const char *name, + struct cfl_variant *value) +{ + struct cfl_kvpair *pair; + + if (!value) { + return -1; + } + + pair = fts_find_pair(metadata, name, strlen(name)); + if (!pair) { + if (cfl_kvlist_insert(metadata->data.as_kvlist, (char *) name, value) != 0) { + cfl_variant_destroy(value); + return -1; + } + return 0; + } + + cfl_variant_destroy(pair->val); + pair->val = value; + cfl_container_move_variant_to_kvlist(metadata->data.as_kvlist, value); + + return 0; +} + +static struct cfl_object *empty_object(void) +{ + struct cfl_kvlist *map; + struct cfl_variant *v; + struct cfl_object *obj; + + map = cfl_kvlist_create(); + if (!map) { + return NULL; + } + + v = cfl_variant_create_from_kvlist(map); + if (!v) { + cfl_kvlist_destroy(map); + return NULL; + } + + obj = object_from_variant(v); + if (!obj) { + cfl_variant_destroy(v); + } + + return obj; +} + +static int append_marker(struct flb_mp_chunk_cobj *out, int type, + struct cfl_object *attributes, struct cfl_object *metadata, + struct flb_mp_chunk_record **marker) +{ + struct flb_mp_chunk_record *record; + + record = flb_mp_chunk_record_create(out); + if (!record) { + return -1; + } + record->event.timestamp.tm.tv_sec = type; + + if (type == FLB_LOG_EVENT_GROUP_START) { + record->cobj_record = copy_object(attributes); + record->cobj_metadata = copy_object(metadata); + } + else { + /* Later processors expect decoded group ends to carry empty maps. */ + record->cobj_record = empty_object(); + record->cobj_metadata = empty_object(); + } + + if (!record->cobj_record || !record->cobj_metadata) { + flb_mp_chunk_cobj_record_destroy(NULL, record); + return -1; + } + + if (type == FLB_LOG_EVENT_GROUP_START) { + record->cobj_group_attributes = record->cobj_record; + record->cobj_group_metadata = record->cobj_metadata; + } + + cfl_list_add(&record->_head, &out->records); + if (marker) { + *marker = record; + } + + return 0; +} + +/* MessagePack wire size, separate from conservative policy-memory accounting. */ +static size_t wire_value_size(struct cfl_variant *value, int depth) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + size_t size; + size_t child; + size_t count; + size_t index; + int64_t signed_value; + uint64_t unsigned_value; + + if (!value || depth > FTS_MAX_DEPTH) { + return SIZE_MAX; + } + + switch (value->type) { + case CFL_VARIANT_NULL: + case CFL_VARIANT_BOOL: + return 1; + case CFL_VARIANT_DOUBLE: + return 9; + case CFL_VARIANT_INT: + signed_value = value->data.as_int64; + if (signed_value < 0) { + if (signed_value >= -32) { + return 1; + } + else if (signed_value >= INT8_MIN) { + return 2; + } + else if (signed_value >= INT16_MIN) { + return 3; + } + else if (signed_value >= INT32_MIN) { + return 5; + } + return 9; + } + unsigned_value = signed_value; + break; + case CFL_VARIANT_UINT: + unsigned_value = value->data.as_uint64; + break; + case CFL_VARIANT_STRING: + size = cfl_variant_size_get(value); + if (size > SIZE_MAX - 5) { + return SIZE_MAX; + } + if (size < 32) { + return size + 1; + } + else if (size <= UINT8_MAX) { + return size + 2; + } + else if (size <= UINT16_MAX) { + return size + 3; + } + return size + 5; + case CFL_VARIANT_BYTES: + size = cfl_variant_size_get(value); + if (size > SIZE_MAX - 5) { + return SIZE_MAX; + } + if (size <= UINT8_MAX) { + return size + 2; + } + else if (size <= UINT16_MAX) { + return size + 3; + } + return size + 5; + case CFL_VARIANT_ARRAY: + case CFL_VARIANT_KVLIST: + if (value->type == CFL_VARIANT_ARRAY) { + count = value->data.as_array->entry_count; + } + else { + count = cfl_kvlist_count(value->data.as_kvlist); + } + + /* The CFL serializer reserves a map32 header even for small maps. */ + if (value->type == CFL_VARIANT_KVLIST) { + size = 5; + } + else if (count < 16) { + size = 1; + } + else if (count <= UINT16_MAX) { + size = 3; + } + else { + size = 5; + } + + if (value->type == CFL_VARIANT_ARRAY) { + for (index = 0; index < count; index++) { + child = wire_value_size(value->data.as_array->entries[index], depth + 1); + if (child > SIZE_MAX - size) { + return SIZE_MAX; + } + size += child; + } + } + else { + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + + /* key: string header plus bytes */ + child = cfl_sds_len(pair->key); + if (size > SIZE_MAX - 5 || child > SIZE_MAX - size - 5) { + return SIZE_MAX; + } + if (child < 32) { + size += child + 1; + } + else if (child <= UINT8_MAX) { + size += child + 2; + } + else if (child <= UINT16_MAX) { + size += child + 3; + } + else { + size += child + 5; + } + + /* value */ + child = wire_value_size(pair->val, depth + 1); + if (child > SIZE_MAX - size) { + return SIZE_MAX; + } + size += child; + } + } + return size; + default: + return SIZE_MAX; + } + + /* Non-negative integers */ + if (unsigned_value <= 127) { + return 1; + } + else if (unsigned_value <= UINT8_MAX) { + return 2; + } + else if (unsigned_value <= UINT16_MAX) { + return 3; + } + else if (unsigned_value <= UINT32_MAX) { + return 5; + } + return 9; +} + +struct borrowed_record { + struct flb_mp_chunk_record *source; + struct flb_mp_chunk_record *result; + int body; + int metadata; + int pooled; + struct cfl_list _head; +}; + +#define BORROW_BLOCK_ENTRIES 64 +#define BORROW_BLOCK_LIMIT 8 + +struct borrowed_record_block { + struct borrowed_record_block *next; + size_t used; + struct borrowed_record records[BORROW_BLOCK_ENTRIES]; +}; + +struct borrowed_record_pool { + struct borrowed_record_block *blocks; + size_t count; +}; + +/* This pool owns bookkeeping only, never policy values or borrowed objects. */ +static struct borrowed_record *borrow_record_create(struct borrowed_record_pool *pool) +{ + struct borrowed_record_block *block; + struct borrowed_record *record; + + block = pool->blocks; + if ((!block || block->used == BORROW_BLOCK_ENTRIES) && pool->count < BORROW_BLOCK_LIMIT) { + /* On failure fall back to an individual heap allocation below */ + block = flb_calloc(1, sizeof(struct borrowed_record_block)); + if (block) { + block->next = pool->blocks; + pool->blocks = block; + pool->count++; + } + } + + if (block && block->used < BORROW_BLOCK_ENTRIES) { + record = &block->records[block->used++]; + record->pooled = FLB_TRUE; + return record; + } + + record = flb_calloc(1, sizeof(struct borrowed_record)); + if (!record) { + flb_errno(); + } + + return record; +} + +/* Ownership moves only after the entire staged chunk has been constructed. */ +static void release_borrowed_records(struct cfl_list *records, + struct borrowed_record_pool *pool, int commit) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct borrowed_record *record; + struct borrowed_record_block *block; + + cfl_list_foreach_safe(head, tmp, records) { + record = cfl_list_entry(head, struct borrowed_record, _head); + + if (record->body) { + if (commit) { + record->source->cobj_record = NULL; + } + else { + record->result->cobj_record = NULL; + } + } + + if (record->metadata) { + if (commit) { + record->source->cobj_metadata = NULL; + } + else { + record->result->cobj_metadata = NULL; + } + } + + cfl_list_del(head); + if (!record->pooled) { + flb_free(record); + } + } + + while (pool->blocks) { + block = pool->blocks; + pool->blocks = block->next; + flb_free(block); + } + pool->count = 0; +} + +/* + * A following filter forces serialization; a following native processor may + * mutate CFL containers and therefore requires ordinary heap-backed values. + */ +static int serializes_after_callback(struct flb_processor_instance *ins) +{ + struct flb_processor *parent = ins->pu->parent; + struct mk_list *next = ins->pu->_head.next; + struct flb_processor_unit *unit; + + if (next == &parent->logs) { + return FLB_TRUE; + } + + unit = mk_list_entry(next, struct flb_processor_unit, _head); + return unit->unit_type == FLB_PROCESSOR_UNIT_FILTER; +} + +int fts_process_logs(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + struct cfl_list borrowed; + struct borrowed_record_pool borrow_pool = {0}; + struct borrowed_record *borrow; + struct fts_runtime *ctx = ins->context; + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_cobj *out = NULL; + struct flb_mp_chunk_record *record; + struct flb_mp_chunk_record *next; + struct flb_mp_chunk_record *group = NULL; + struct flb_mp_chunk_record *last_group = NULL; + struct cfl_variant *last_source_id = NULL; + struct cfl_kvpair *source_id; + struct cfl_kvpair *last_resource; + struct cfl_kvpair *current_resource; + struct cfl_kvpair *last_identity; + int resource_id = 0; + struct cfl_object *source_attrs = NULL; + struct cfl_object *source_meta = NULL; + struct cfl_object attrs; + struct cfl_object meta; + struct record_call call; + struct flb_condition *condition; + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_variant *body; + struct cfl_variant *metadata; + struct fts_value *context_attrs; + struct fts_value *context_meta; + lua_State *l = ctx->vm->state; + int ret; + int type; + int group_number = 0; + int emitted_group_number = -1; + int selected; + int base; + int emitted_id = 0; + int errors = 0; + size_t undo_mark; + struct cfl_variant *id_value; + size_t output_bytes = 0; + size_t cost; + + ctx->tag_ready = FLB_FALSE; + cfl_list_init(&borrowed); + + chunk->record_pool_enabled = FLB_TRUE; + if (serializes_after_callback(ins)) { + flb_mp_chunk_cobj_value_pool_enable(chunk); + } + + /* Decode every record of the chunk, ignoring the condition for now */ + condition = chunk->condition; + chunk->condition = NULL; + while ((ret = flb_mp_chunk_cobj_record_next(chunk, &record)) == FLB_MP_CHUNK_RECORD_OK) { + } + chunk->condition = condition; + chunk->record_pos = NULL; + if (ret != FLB_MP_CHUNK_RECORD_EOF) { + goto failure; + } + + /* Staged output chunk; it replaces the original records on success */ + out = flb_calloc(1, sizeof(struct flb_mp_chunk_cobj)); + if (!out) { + flb_errno(); + goto failure; + } + cfl_list_init(&out->records); + out->record_pool_enabled = FLB_TRUE; + + cfl_list_foreach(head, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + + /* Group markers update the source context and are not copied as-is */ + if (flb_log_event_decoder_get_record_type(&record->event, &type) != 0) { + goto failure; + } + + if (type == FLB_LOG_EVENT_GROUP_START) { + /* A nested start establishes a new flat transport context. */ + source_attrs = record->cobj_record; + source_meta = record->cobj_metadata; + group_number++; + continue; + } + + if (type == FLB_LOG_EVENT_GROUP_END) { + /* Stray ends carry no record and do not invalidate healthy siblings. */ + source_attrs = NULL; + source_meta = NULL; + continue; + } + + /* Defaults for records that are not selected or keep their values */ + selected = !condition || flb_condition_evaluate(condition, record); + memset(&attrs, 0, sizeof(attrs)); + memset(&meta, 0, sizeof(meta)); + attrs.variant = source_attrs ? source_attrs->variant : NULL; + meta.variant = source_meta ? source_meta->variant : NULL; + base = lua_gettop(l); + ret = 0; + body = NULL; + metadata = NULL; + + /* Run the policy for selected records */ + if (selected) { + memset(&call, 0, sizeof(call)); + call.ctx = ctx; + call.chunk = chunk; + call.record = record; + call.attributes = source_attrs; + call.metadata = source_meta; + call.tag = tag; + call.tag_len = tag_len; + + /* + * Plain records edit the chunk's heap tree in place; the undo log + * keeps the whole chunk reversible until its output is final. + */ + undo_mark = ctx->undo_count; + ctx->inplace_ok = !source_attrs && ctx->undo_held <= ctx->memory_limit; + ret = fts_runtime_call(ctx, &call, 7); + ctx->inplace_ok = FLB_FALSE; + ctx->native_bytes -= ctx->undo_charged; + ctx->undo_charged = 0; + + if (ret != 0) { + fts_undo_rollback(ctx, undo_mark); + errors++; + if (errors <= 5) { + flb_plg_error(ins, "log policy failed: %.256s; edits rolled back", + lua_type(l, -1) == LUA_TSTRING ? + lua_tostring(l, -1) : "non-string error"); + } + if (strcmp(ctx->error_mode, "drop") == 0) { + lua_settop(l, base); + continue; + } + } + else if (lua_type(l, base + 1) == LUA_TSTRING && + strcmp(lua_tostring(l, base + 1), "drop") == 0) { + lua_settop(l, base); + continue; + } + else { + body = call.result_body; + metadata = call.result_metadata; + context_attrs = lua_touserdata(l, base + 4); + context_meta = lua_touserdata(l, base + 5); + if (context_attrs) { + attrs.variant = context_attrs->owned; + } + if (context_meta) { + meta.variant = context_meta->owned; + } + } + } + + /* Close the open group when the record's context no longer matches it */ + if (group && (!attrs.variant || emitted_group_number != group_number || + !values_equal(group->cobj_record->variant, attrs.variant) || + !(has_otel_context(group->cobj_metadata) ? + context_metadata_equal(group->cobj_metadata->variant, meta.variant) : + values_equal(group->cobj_metadata->variant, meta.variant)))) { + if (append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + lua_settop(l, base); + goto failure; + } + group = NULL; + } + + /* Open a new group, charging its marker and context to the output limit */ + if (attrs.variant && !group) { + if (output_bytes > ctx->output_limit || ctx->output_limit - output_bytes < 32) { + lua_settop(l, base); + goto failure; + } + output_bytes += 32; + + cost = wire_value_size(attrs.variant, 0); + if (cost > ctx->output_limit || output_bytes > ctx->output_limit - cost) { + lua_settop(l, base); + goto failure; + } + output_bytes += cost; + + cost = wire_value_size(meta.variant, 0); + if (cost > ctx->output_limit || output_bytes > ctx->output_limit - cost) { + lua_settop(l, base); + goto failure; + } + output_bytes += cost; + + attrs.type = CFL_OBJECT_KVLIST; + meta.type = CFL_OBJECT_KVLIST; + if (append_marker(out, FLB_LOG_EVENT_GROUP_START, &attrs, &meta, &group) != 0) { + lua_settop(l, base); + goto failure; + } + + /* OTLP groups get regenerated resource_id/scope_id transport IDs */ + if (has_otel_context(group->cobj_metadata)) { + /* Groups without a source ID never share a resource with others. */ + source_id = fts_find_pair(source_meta->variant, "resource_id", 11); + last_resource = last_group ? + fts_find_pair(last_group->cobj_record->variant, "resource", 8) : + NULL; + current_resource = fts_find_pair(group->cobj_record->variant, "resource", 8); + + if (source_id && last_group && last_source_id && + values_equal(last_source_id, source_id->val) && + context_kind_equal(last_resource ? last_resource->val : NULL, + current_resource ? current_resource->val : NULL)) { + last_identity = fts_find_pair(last_group->cobj_metadata->variant, + "resource_id", 11); + id_value = fts_copy_value(last_identity->val, 0); + } + else { + id_value = cfl_variant_create_from_int64(resource_id++); + } + + if (set_transport_id(group->cobj_metadata->variant, "resource_id", + id_value) != 0) { + lua_settop(l, base); + goto failure; + } + + id_value = cfl_variant_create_from_int64(emitted_id++); + if (set_transport_id(group->cobj_metadata->variant, "scope_id", id_value) != 0) { + lua_settop(l, base); + goto failure; + } + + last_group = group; + last_source_id = source_id ? source_id->val : NULL; + } + else { + last_group = NULL; + last_source_id = NULL; + } + emitted_group_number = group_number; + } + + if (!attrs.variant) { + last_group = NULL; + last_source_id = NULL; + } + + /* Charge the record framing, body and metadata to the output limit */ + if (output_bytes > ctx->output_limit || ctx->output_limit - output_bytes < 16) { + lua_settop(l, base); + goto failure; + } + output_bytes += 16; + + cost = wire_value_size(body ? body : record->cobj_record->variant, 0); + if (cost > ctx->output_limit || output_bytes > ctx->output_limit - cost) { + lua_settop(l, base); + goto failure; + } + output_bytes += cost; + + cost = wire_value_size(metadata ? metadata : record->cobj_metadata->variant, 0); + if (cost > ctx->output_limit || output_bytes > ctx->output_limit - cost) { + lua_settop(l, base); + goto failure; + } + output_bytes += cost; + + /* + * Build the output record: changed roots are copied into the chunk, + * unchanged roots are borrowed from the source record. + */ + next = flb_mp_chunk_record_create(out); + if (!next) { + lua_settop(l, base); + goto failure; + } + next->event = record->event; + next->cobj_record = body ? result_object(chunk, body) : record->cobj_record; + next->cobj_metadata = metadata ? result_object(chunk, metadata) : record->cobj_metadata; + + if (selected && ret == 0 && + (record->event.timestamp.tm.tv_sec != (time_t) lua_tonumber(l, base + 6) || + record->event.timestamp.tm.tv_nsec != (long) lua_tonumber(l, base + 7))) { + next->event.timestamp.tm.tv_sec = (time_t) lua_tonumber(l, base + 6); + next->event.timestamp.tm.tv_nsec = (long) lua_tonumber(l, base + 7); + next->event.raw_timestamp = NULL; + } + + if (!next->cobj_record || !next->cobj_metadata) { + if (!body) { + next->cobj_record = NULL; + } + if (!metadata) { + next->cobj_metadata = NULL; + } + flb_mp_chunk_cobj_record_destroy(NULL, next); + lua_settop(l, base); + goto failure; + } + + /* Track borrowed roots so ownership can be settled at commit/rollback */ + if (!body || !metadata) { + borrow = borrow_record_create(&borrow_pool); + if (!borrow) { + if (!body) { + next->cobj_record = NULL; + } + if (!metadata) { + next->cobj_metadata = NULL; + } + flb_mp_chunk_cobj_record_destroy(NULL, next); + lua_settop(l, base); + goto failure; + } + borrow->source = record; + borrow->result = next; + borrow->body = !body; + borrow->metadata = !metadata; + cfl_list_add(&borrow->_head, &borrowed); + } + + next->cobj_group_attributes = group ? group->cobj_record : NULL; + next->cobj_group_metadata = group ? group->cobj_metadata : NULL; + cfl_list_add(&next->_head, &out->records); + lua_settop(l, base); + } + + /* Close an unterminated transport group after its last retained record. */ + if (group && append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + goto failure; + } + + /* Commit: drop the source records and move the staged ones into the chunk */ + fts_undo_commit(ctx); + release_borrowed_records(&borrowed, &borrow_pool, FLB_TRUE); + + cfl_list_foreach_safe(head, tmp, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + flb_mp_chunk_cobj_record_destroy(NULL, record); + } + + cfl_list_foreach_safe(head, tmp, &out->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + cfl_list_del(head); + cfl_list_add(head, &chunk->records); + } + + chunk->active_group_attributes = NULL; + chunk->active_group_metadata = NULL; + flb_mp_chunk_cobj_destroy(out); + + if (errors) { + fts_report_policy_error(ins); + if (errors > 5) { + flb_plg_warn(ins, "%d additional policy errors in this chunk", errors - 5); + } + } + + ctx->instructions_left = ctx->instruction_limit; + fts_collect(ctx, FLB_FALSE); + + return FLB_PROCESSOR_SUCCESS; + +failure: + fts_undo_rollback(ctx, 0); + release_borrowed_records(&borrowed, &borrow_pool, FLB_FALSE); + flb_mp_chunk_cobj_destroy(out); + ctx->instructions_left = ctx->instruction_limit; + fts_collect(ctx, FLB_FALSE); + + flb_plg_error(ins, "cannot construct policy output; edits discarded"); + if (strcmp(ctx->error_mode, "drop") == 0) { + cfl_list_foreach_safe(head, tmp, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + flb_mp_chunk_cobj_record_destroy(NULL, record); + } + chunk->active_group_attributes = NULL; + chunk->active_group_metadata = NULL; + fts_report_policy_error(ins); + return FLB_PROCESSOR_SUCCESS; + } + + return FLB_PROCESSOR_FAILURE; +} + +static int register_log_helpers(lua_State *l) +{ + if (fts_load_helpers(l, fts_log_helpers, sizeof(fts_log_helpers) - 1, + "fluentbit.log_helpers") != 0) { + return -1; + } + + return 0; +} + +const struct fts_signal_adapter fts_logs_adapter = { + .event_type = FLB_PROCESSOR_LOGS, + .register_helpers = register_log_helpers, + .run_callback = run_record +}; diff --git a/plugins/processor_fts/fts_metrics.c b/plugins/processor_fts/fts_metrics.c new file mode 100644 index 00000000000..0b46e19abd6 --- /dev/null +++ b/plugins/processor_fts/fts_metrics.c @@ -0,0 +1,572 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fts_telemetry.h" +#include + +const struct fts_signal_adapter fts_metrics_adapter = { + .event_type = FLB_PROCESSOR_METRICS, + .register_helpers = fts_register_telemetry_helpers, + .run_callback = fts_run_telemetry +}; + +static struct cfl_kvlist *map_at(struct cfl_kvlist *map, char *key) +{ + struct cfl_variant *value; + struct cfl_kvlist *child; + + value = cfl_kvlist_fetch(map, key); + if (value) { + return value->type == CFL_VARIANT_KVLIST ? value->data.as_kvlist : NULL; + } + + child = cfl_kvlist_create(); + if (!child || cfl_kvlist_insert_kvlist(map, key, child) != 0) { + cfl_kvlist_destroy(child); + return NULL; + } + + return child; +} + +static int copy_map(struct cfl_kvlist *dst, struct cfl_kvlist *src) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_variant *copy; + + cfl_list_foreach(head, &src->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + copy = fts_copy_value(pair->val, 0); + if (!copy || cfl_kvlist_insert_s(dst, pair->key, cfl_sds_len(pair->key), copy) != 0) { + cfl_variant_destroy(copy); + return -1; + } + } + + return 0; +} + +static int family_metadata(struct cmt *out, struct cmt *source, struct cmt_map *family, + const char *type, size_t resource_index) +{ + struct cfl_kvlist *metadata; + + if (!family->otlp_metadata) { + family->otlp_metadata = cfl_kvlist_create(); + } + + if (!family->otlp_metadata) { + return -1; + } + + metadata = map_at(family->otlp_metadata, "routing"); + if (!metadata) { + return -1; + } + + cfl_kvlist_remove(metadata, "scope_flat_index"); + cfl_kvlist_remove(metadata, "resource_index"); + cfl_kvlist_remove(metadata, "scope_index"); + + if (cfl_kvlist_insert_uint64(metadata, "resource_index", resource_index) != 0 || + cfl_kvlist_insert_uint64(metadata, "scope_index", 0) != 0) { + return -1; + } + + return 0; +} + +/* Move complete families: concatenation would aggregate samples and lose identity. */ +#define MOVE_FAMILIES(member, family_type, family, type_name) \ + cfl_list_foreach_safe(head, tmp, &source->member) { \ + family = cfl_list_entry(head, family_type, _head); \ + if (family_metadata(out, source, family->map, type_name, index) != 0) { \ + goto failure; \ + } \ + cfl_list_del(head); \ + cfl_list_add(head, &out->member); \ + family->cmt = out; \ + } + +static struct cmt *merge_contexts(struct cfl_list *contexts, struct cmt *input) +{ + struct cmt *out; + struct cmt *source; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; + struct cmt_summary *summary; + struct cmt_untyped *untyped; + struct cfl_list *context_head; + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_array *resources; + struct cfl_kvlist *resource; + struct cfl_variant *value; + struct cfl_variant *copy; + const char *keys[] = {"resource", "resource_metrics", "scope", "scope_metrics"}; + size_t i; + size_t index = 0; + + out = cmt_create(); + if (!out) { + return NULL; + } + + if (copy_map(out->internal_metadata, input->internal_metadata) != 0 || + copy_map(out->external_metadata, input->external_metadata) != 0) { + goto failure; + } + + cfl_kvlist_remove(out->external_metadata, "resource_metrics_list"); + cfl_kvlist_remove(out->external_metadata, "otlp"); + + resources = cfl_array_create(cfl_list_size(contexts)); + if (!resources || + cfl_kvlist_insert_array(out->external_metadata, "resource_metrics_list", + resources) != 0) { + cfl_array_destroy(resources); + goto failure; + } + + cfl_list_foreach(context_head, contexts) { + source = cfl_list_entry(context_head, struct cmt, _head); + + resource = cfl_kvlist_create(); + if (!resource || cfl_array_append_kvlist(resources, resource) != 0) { + cfl_kvlist_destroy(resource); + goto failure; + } + + for (i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { + value = cfl_kvlist_fetch(source->external_metadata, (char *) keys[i]); + if (value) { + copy = fts_copy_value(value, 0); + if (!copy || cfl_kvlist_insert(resource, (char *) keys[i], copy) != 0) { + cfl_variant_destroy(copy); + goto failure; + } + } + } + + MOVE_FAMILIES(counters, struct cmt_counter, counter, "counter"); + MOVE_FAMILIES(gauges, struct cmt_gauge, gauge, "gauge"); + MOVE_FAMILIES(histograms, struct cmt_histogram, histogram, "histogram"); + MOVE_FAMILIES(exp_histograms, struct cmt_exp_histogram, exp_histogram, "exp_histogram"); + MOVE_FAMILIES(summaries, struct cmt_summary, summary, "summary"); + MOVE_FAMILIES(untypeds, struct cmt_untyped, untyped, "untyped"); + index++; + } + + return out; + +failure: + cmt_destroy(out); + return NULL; +} + +/* OTLP scalars are int64 or double: uint64 values must fit int64 exactly. */ +static int scalar_value_supported(struct cmt_metric *sample) +{ + if (cmt_metric_get_value_type(sample) != CMT_METRIC_VALUE_UINT64) { + return FLB_TRUE; + } + + return cmt_metric_get_uint64_value(sample) <= INT64_MAX; +} + +static int scalar_values_supported(struct cmt_map *map) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + if (map->metric_static_set && !scalar_value_supported(&map->metric)) { + return FLB_FALSE; + } + + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (!scalar_value_supported(sample)) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + +struct unsupported_family { + struct cfl_list *head; + struct cfl_list *next; + struct cmt_map *map; + void *family; + int type; + struct cfl_kvlist *metadata; +}; + +/* Families are detached only while the input is encoded, then restored. */ +#define DETACH_UNSUPPORTED(member, family_type, item, map_type) \ + cfl_list_foreach_safe(head, next, &input->member) { \ + item = cfl_list_entry(head, family_type, _head); \ + if (!scalar_values_supported(item->map)) { \ + rejected[rejected_count].head = head; \ + rejected[rejected_count].next = head->next; \ + rejected[rejected_count].map = item->map; \ + rejected[rejected_count].family = item; \ + rejected[rejected_count].type = map_type; \ + rejected_count++; \ + cfl_list_del(head); \ + } \ + } + +static uint64_t routing_value(struct cfl_kvlist *routing, char *key) +{ + struct cfl_variant *value; + + value = routing ? cfl_kvlist_fetch(routing, key) : NULL; + if (value && value->type == CFL_VARIANT_UINT) { + return value->data.as_uint64; + } + + if (value && value->type == CFL_VARIANT_INT && value->data.as_int64 >= 0) { + return (uint64_t) value->data.as_int64; + } + + return 0; +} + +/* Preserve the original contexts of families that cannot enter the OTLP API. */ +static int prepare_rejected_families(struct cmt *out, struct cmt *input, + struct unsupported_family *families, size_t count) +{ + struct cfl_variant *output_resources; + struct cfl_variant *input_resources; + struct cfl_variant *copy; + struct cfl_variant *scopes; + struct cfl_variant *flat; + struct cfl_kvlist *resource; + struct cfl_kvlist *routing; + const char *keys[] = {"resource", "resource_metrics", "scope", "scope_metrics"}; + size_t offset; + size_t index; + size_t key_index; + size_t resource_index; + size_t scope_index; + uint64_t flat_index; + size_t scope_count; + + output_resources = cfl_kvlist_fetch(out->external_metadata, "resource_metrics_list"); + if (!output_resources || output_resources->type != CFL_VARIANT_ARRAY) { + return -1; + } + + cfl_array_resizable(output_resources->data.as_array, CFL_TRUE); + offset = output_resources->data.as_array->entry_count; + + input_resources = cfl_kvlist_fetch(input->external_metadata, "resource_metrics_list"); + if (input_resources && input_resources->type == CFL_VARIANT_ARRAY && + input_resources->data.as_array->entry_count > 0) { + for (index = 0; index < input_resources->data.as_array->entry_count; index++) { + copy = fts_copy_value(input_resources->data.as_array->entries[index], 0); + if (!copy || cfl_array_append(output_resources->data.as_array, copy) != 0) { + cfl_variant_destroy(copy); + return -1; + } + } + } + else { + resource = cfl_kvlist_create(); + if (!resource || cfl_array_append_kvlist(output_resources->data.as_array, resource) != 0) { + cfl_kvlist_destroy(resource); + return -1; + } + + for (key_index = 0; key_index < sizeof(keys) / sizeof(keys[0]); key_index++) { + copy = cfl_kvlist_fetch(input->external_metadata, (char *) keys[key_index]); + if (copy) { + copy = fts_copy_value(copy, 0); + if (!copy || cfl_kvlist_insert(resource, (char *) keys[key_index], copy) != 0) { + cfl_variant_destroy(copy); + return -1; + } + } + } + } + + for (index = 0; index < count; index++) { + if (families[index].map->otlp_metadata) { + families[index].metadata = + cmt_otlp_metadata_clone(families[index].map->otlp_metadata); + } + else { + families[index].metadata = cfl_kvlist_create(); + } + + if (!families[index].metadata) { + return -1; + } + + routing = map_at(families[index].metadata, "routing"); + if (!routing) { + return -1; + } + + resource_index = routing_value(routing, "resource_index"); + scope_index = routing_value(routing, "scope_index"); + + flat = cfl_kvlist_fetch(routing, "scope_flat_index"); + if (flat && input_resources && input_resources->type == CFL_VARIANT_ARRAY) { + flat_index = routing_value(routing, "scope_flat_index"); + for (resource_index = 0; + resource_index < input_resources->data.as_array->entry_count; + resource_index++) { + copy = input_resources->data.as_array->entries[resource_index]; + if (copy->type == CFL_VARIANT_KVLIST) { + scopes = cfl_kvlist_fetch(copy->data.as_kvlist, "scope_metrics_list"); + } + else { + scopes = NULL; + } + + if (scopes && scopes->type == CFL_VARIANT_ARRAY && + scopes->data.as_array->entry_count) { + scope_count = scopes->data.as_array->entry_count; + } + else { + scope_count = 1; + } + + if (flat_index < scope_count) { + scope_index = flat_index; + break; + } + flat_index -= scope_count; + } + } + + if (resource_index >= output_resources->data.as_array->entry_count - offset) { + resource_index = 0; + scope_index = 0; + } + + cfl_kvlist_remove(routing, "scope_flat_index"); + cfl_kvlist_remove(routing, "resource_index"); + cfl_kvlist_remove(routing, "scope_index"); + + if (cfl_kvlist_insert_uint64(routing, "resource_index", offset + resource_index) != 0 || + cfl_kvlist_insert_uint64(routing, "scope_index", scope_index) != 0) { + return -1; + } + } + + return 0; +} + +/* The input context stays owned by its producer, so kept families are copied. */ +static int copy_rejected_families(struct cmt *out, struct unsupported_family *families, + size_t count) +{ + int ret; + size_t index; + struct cfl_list *list; + struct cfl_list *head; + struct cmt_map *map; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + + for (index = 0; index < count; index++) { + if (families[index].type == CMT_COUNTER) { + ret = cmt_cat_counter(out, families[index].family, NULL); + list = &out->counters; + } + else if (families[index].type == CMT_GAUGE) { + ret = cmt_cat_gauge(out, families[index].family, NULL); + list = &out->gauges; + } + else { + ret = cmt_cat_untyped(out, families[index].family, NULL); + list = &out->untypeds; + } + + if (ret != 0) { + return -1; + } + + /* Attach the routing metadata prepared for this family to its copy. */ + map = NULL; + cfl_list_foreach(head, list) { + if (families[index].type == CMT_COUNTER) { + counter = cfl_list_entry(head, struct cmt_counter, _head); + map = counter->map; + } + else if (families[index].type == CMT_GAUGE) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + map = gauge->map; + } + else { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + map = untyped->map; + } + + if (strcmp(map->opts->fqname, families[index].map->opts->fqname) == 0) { + break; + } + map = NULL; + } + + if (!map) { + return -1; + } + + cfl_kvlist_destroy(map->otlp_metadata); + map->otlp_metadata = families[index].metadata; + families[index].metadata = NULL; + } + + return 0; +} + +int fts_process_metrics(struct flb_processor_instance *ins, struct cmt *input, + struct cmt **output, const char *tag, int tag_len) +{ + struct fts_runtime *ctx = ins->context; + cfl_sds_t encoded; + char *buffer = NULL; + size_t size; + size_t offset = 0; + struct cfl_list contexts; + struct cfl_list *head; + struct cfl_list *next; + struct unsupported_family *rejected = NULL; + size_t rejected_count = 0; + size_t family_count; + size_t index; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + int ret; + int handled_errors; + const char *phase = "encode native metrics"; + + *output = NULL; + cfl_list_init(&contexts); + + family_count = cfl_list_size(&input->counters) + cfl_list_size(&input->gauges) + + cfl_list_size(&input->untypeds); + if (family_count > 0) { + rejected = flb_calloc(family_count, sizeof(*rejected)); + if (!rejected) { + flb_errno(); + goto failure; + } + } + + DETACH_UNSUPPORTED(counters, struct cmt_counter, counter, CMT_COUNTER); + DETACH_UNSUPPORTED(gauges, struct cmt_gauge, gauge, CMT_GAUGE); + DETACH_UNSUPPORTED(untypeds, struct cmt_untyped, untyped, CMT_UNTYPED); + + encoded = cmt_encode_opentelemetry_create(input); + + /* Restore the exact input list order before invoking any policy. */ + for (index = rejected_count; index > 0; index--) { + cfl_list_add(rejected[index - 1].head, rejected[index - 1].next); + } + + if (!encoded) { + goto failure; + } + + phase = "transform metric policies"; + ret = fts_telemetry_transform(ins, encoded, cfl_sds_len(encoded), &buffer, &size, + tag, tag_len); + cmt_encode_opentelemetry_destroy(encoded); + handled_errors = ret > 0; + if (ret < 0) { + goto failure; + } + + phase = "decode transformed metrics"; + ret = size ? cmt_decode_opentelemetry_create(&contexts, buffer, size, &offset) : 0; + flb_free(buffer); + buffer = NULL; + if (ret != 0) { + goto failure; + } + + phase = "merge native metric contexts"; + *output = merge_contexts(&contexts, input); + cmt_decode_opentelemetry_destroy(&contexts); + if (!*output) { + goto failure; + } + + if (rejected_count > 0) { + if (strcmp(ctx->error_mode, "keep") == 0) { + phase = "restore native uint64 family contexts"; + if (prepare_rejected_families(*output, input, rejected, rejected_count) != 0) { + cmt_destroy(*output); + *output = NULL; + goto failure; + } + + if (copy_rejected_families(*output, rejected, rejected_count) != 0) { + cmt_destroy(*output); + *output = NULL; + goto failure; + } + } + flb_plg_warn(ins, "FTS cannot process %zu native uint64 scalar families above " + "INT64_MAX; error_mode applied to these families", rejected_count); + } + + if (handled_errors || rejected_count > 0) { + fts_report_policy_error(ins); + } + + flb_free(rejected); + return FLB_PROCESSOR_SUCCESS; + +failure: + for (index = 0; index < rejected_count; index++) { + cfl_kvlist_destroy(rejected[index].metadata); + } + flb_free(rejected); + + fts_report_policy_error(ins); + flb_free(buffer); + cmt_decode_opentelemetry_destroy(&contexts); + flb_plg_error(ins, "cannot construct FTS metrics output (%s); edits discarded", phase); + + *output = strcmp(ctx->error_mode, "drop") == 0 ? cmt_create() : input; + return *output ? FLB_PROCESSOR_SUCCESS : FLB_PROCESSOR_FAILURE; +} diff --git a/plugins/processor_fts/fts_proto.c b/plugins/processor_fts/fts_proto.c new file mode 100644 index 00000000000..dfcb1b77ed0 --- /dev/null +++ b/plugins/processor_fts/fts_proto.c @@ -0,0 +1,616 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "fts_proto.h" + +/* Reflection is confined to the OTLP schema; policies see typed telemetry values. */ +static int reserve(struct fts_proto_budget *budget, size_t size) +{ + if (size > budget->limit || budget->used > budget->limit - size) { + return -1; + } + + budget->used += size; + return 0; +} + +void *fts_proto_alloc(void *data, size_t size) +{ + void *ptr; + + if (reserve(data, size) != 0) { + return NULL; + } + + ptr = calloc(1, size ? size : 1); + if (!ptr) { + flb_errno(); + } + + return ptr; +} + +void fts_proto_free(void *data, void *ptr) +{ + free(ptr); +} + +static size_t scalar_size(ProtobufCType type) +{ + switch (type) { + case PROTOBUF_C_TYPE_MESSAGE: + case PROTOBUF_C_TYPE_STRING: + return sizeof(void *); + case PROTOBUF_C_TYPE_BYTES: + return sizeof(ProtobufCBinaryData); + case PROTOBUF_C_TYPE_INT64: + case PROTOBUF_C_TYPE_UINT64: + case PROTOBUF_C_TYPE_FIXED64: + case PROTOBUF_C_TYPE_SFIXED64: + case PROTOBUF_C_TYPE_SINT64: + case PROTOBUF_C_TYPE_DOUBLE: + return 8; + default: + return 4; + } +} + +static int is_key_value(const ProtobufCFieldDescriptor *field) +{ + const ProtobufCMessageDescriptor *descriptor; + + if (field->type != PROTOBUF_C_TYPE_MESSAGE) { + return FLB_FALSE; + } + + descriptor = field->descriptor; + return strcmp(descriptor->short_name, "KeyValue") == 0; +} + +static struct cfl_variant *read_scalar(const ProtobufCFieldDescriptor *field, const void *ptr, + struct fts_proto_budget *budget, int depth) +{ + const ProtobufCBinaryData *bytes; + const char *string; + + if (reserve(budget, 256) != 0) { + return NULL; + } + + switch (field->type) { + case PROTOBUF_C_TYPE_MESSAGE: + return fts_proto_read(*(ProtobufCMessage *const *) ptr, budget, depth + 1); + case PROTOBUF_C_TYPE_STRING: + string = *(const char *const *) ptr; + string = string ? string : ""; + if (reserve(budget, strlen(string) + 1) != 0) { + return NULL; + } + return cfl_variant_create_from_string((char *) string); + case PROTOBUF_C_TYPE_BYTES: + bytes = ptr; + if (reserve(budget, bytes->len) != 0) { + return NULL; + } + return cfl_variant_create_from_bytes((char *) bytes->data, bytes->len, CFL_FALSE); + case PROTOBUF_C_TYPE_BOOL: + return cfl_variant_create_from_bool(*(const protobuf_c_boolean *) ptr); + case PROTOBUF_C_TYPE_ENUM: + case PROTOBUF_C_TYPE_INT32: + case PROTOBUF_C_TYPE_SINT32: + case PROTOBUF_C_TYPE_SFIXED32: + return cfl_variant_create_from_int64(*(const int32_t *) ptr); + case PROTOBUF_C_TYPE_UINT32: + case PROTOBUF_C_TYPE_FIXED32: + return cfl_variant_create_from_uint64(*(const uint32_t *) ptr); + case PROTOBUF_C_TYPE_INT64: + case PROTOBUF_C_TYPE_SINT64: + case PROTOBUF_C_TYPE_SFIXED64: + return cfl_variant_create_from_int64(*(const int64_t *) ptr); + case PROTOBUF_C_TYPE_UINT64: + case PROTOBUF_C_TYPE_FIXED64: + return cfl_variant_create_from_uint64(*(const uint64_t *) ptr); + case PROTOBUF_C_TYPE_DOUBLE: + return cfl_variant_create_from_double(*(const double *) ptr); + default: + return NULL; + } +} + +static struct cfl_variant *read_repeated(const ProtobufCFieldDescriptor *field, + const ProtobufCMessage *message, + struct fts_proto_budget *budget, int depth) +{ + size_t count; + size_t i; + const char *items; + struct cfl_variant *result; + struct cfl_variant *child; + struct cfl_kvlist *map; + struct cfl_array *array; + Opentelemetry__Proto__Common__V1__KeyValue *pair; + + count = *(const size_t *) ((const char *) message + field->quantifier_offset); + items = *(const char *const *) ((const char *) message + field->offset); + + if (count > (SIZE_MAX - 256) / 256 || reserve(budget, count * 256 + 256) != 0) { + return NULL; + } + + if (is_key_value(field)) { + map = cfl_kvlist_create(); + if (!map) { + return NULL; + } + result = cfl_variant_create_from_kvlist(map); + if (!result) { + cfl_kvlist_destroy(map); + return NULL; + } + } + else { + array = cfl_array_create(count); + if (!array) { + return NULL; + } + result = cfl_variant_create_from_array(array); + if (!result) { + cfl_array_destroy(array); + return NULL; + } + } + + for (i = 0; i < count; i++) { + if (is_key_value(field)) { + pair = ((Opentelemetry__Proto__Common__V1__KeyValue *const *) items)[i]; + if (!pair || !pair->key) { + goto failure; + } + child = fts_proto_read((ProtobufCMessage *) pair->value, budget, depth + 1); + if (!child || cfl_kvlist_insert(result->data.as_kvlist, pair->key, child) != 0) { + cfl_variant_destroy(child); + goto failure; + } + } + else { + child = read_scalar(field, items + i * scalar_size(field->type), budget, depth); + if (!child || cfl_array_append(result->data.as_array, child) != 0) { + cfl_variant_destroy(child); + goto failure; + } + } + } + + return result; + +failure: + cfl_variant_destroy(result); + return NULL; +} + +struct cfl_variant *fts_proto_read(const ProtobufCMessage *message, + struct fts_proto_budget *budget, int depth) +{ + const ProtobufCMessageDescriptor *descriptor; + const ProtobufCFieldDescriptor *field; + struct cfl_variant *result; + struct cfl_variant *child; + const char *ptr; + size_t i; + int any; + int wrapper; + + if (depth > FTS_MAX_DEPTH || reserve(budget, 256) != 0) { + return NULL; + } + + if (!message) { + return cfl_variant_create_from_null(); + } + + descriptor = message->descriptor; + any = strcmp(descriptor->short_name, "AnyValue") == 0; + wrapper = strcmp(descriptor->short_name, "ArrayValue") == 0 || + strcmp(descriptor->short_name, "KeyValueList") == 0; + + result = cfl_variant_create_from_kvlist(cfl_kvlist_create()); + if (!result) { + return NULL; + } + + for (i = 0; i < descriptor->n_fields; i++) { + field = &descriptor->fields[i]; + ptr = (const char *) message; + + if ((field->flags & PROTOBUF_C_FIELD_FLAG_ONEOF) && + *(const uint32_t *) (ptr + field->quantifier_offset) != field->id) { + continue; + } + + if (field->label == PROTOBUF_C_LABEL_OPTIONAL && field->quantifier_offset && + !(field->flags & PROTOBUF_C_FIELD_FLAG_ONEOF) && + !*(const protobuf_c_boolean *) (ptr + field->quantifier_offset)) { + continue; + } + + if (field->label != PROTOBUF_C_LABEL_REPEATED && field->type == PROTOBUF_C_TYPE_MESSAGE && + !*(const ProtobufCMessage *const *) (ptr + field->offset)) { + continue; + } + + if (field->label == PROTOBUF_C_LABEL_REPEATED) { + child = read_repeated(field, message, budget, depth); + } + else { + child = read_scalar(field, ptr + field->offset, budget, depth); + } + + if (!child) { + cfl_variant_destroy(result); + return NULL; + } + + if (any || wrapper) { + cfl_variant_destroy(result); + return child; + } + + if (reserve(budget, strlen(field->name) + 128) != 0 || + cfl_kvlist_insert(result->data.as_kvlist, (char *) field->name, child) != 0) { + cfl_variant_destroy(child); + cfl_variant_destroy(result); + return NULL; + } + } + + if (any) { + cfl_variant_destroy(result); + return cfl_variant_create_from_null(); + } + + return result; +} + +static int signed_integer(struct cfl_variant *value, int64_t *result) +{ + if (value->type == CFL_VARIANT_INT) { + *result = value->data.as_int64; + return 0; + } + + if (value->type == CFL_VARIANT_UINT && value->data.as_uint64 <= INT64_MAX) { + *result = (int64_t) value->data.as_uint64; + return 0; + } + + return -1; +} + +static int unsigned_integer(struct cfl_variant *value, uint64_t *result) +{ + if (value->type == CFL_VARIANT_UINT) { + *result = value->data.as_uint64; + return 0; + } + + if (value->type == CFL_VARIANT_INT && value->data.as_int64 >= 0) { + *result = (uint64_t) value->data.as_int64; + return 0; + } + + return -1; +} + +static int write_scalar(const ProtobufCFieldDescriptor *field, struct cfl_variant *value, + void *ptr, struct fts_proto_budget *budget, int depth) +{ + int64_t signed_value; + uint64_t unsigned_value; + size_t size; + char *string; + ProtobufCBinaryData *bytes; + + switch (field->type) { + case PROTOBUF_C_TYPE_MESSAGE: + *(ProtobufCMessage **) ptr = fts_proto_build(field->descriptor, value, budget, depth + 1); + return *(ProtobufCMessage **) ptr ? 0 : -1; + case PROTOBUF_C_TYPE_STRING: + if (value->type != CFL_VARIANT_STRING) { + return -1; + } + size = cfl_variant_size_get(value); + if (memchr(value->data.as_string, 0, size) || + !fts_valid_utf8(value->data.as_string, size) || reserve(budget, size + 1) != 0) { + return -1; + } + + string = malloc(size + 1); + if (!string) { + flb_errno(); + return -1; + } + + memcpy(string, value->data.as_string, size); + string[size] = 0; + *(char **) ptr = string; + return 0; + case PROTOBUF_C_TYPE_BYTES: + if (value->type != CFL_VARIANT_BYTES) { + return -1; + } + size = cfl_variant_size_get(value); + bytes = ptr; + bytes->data = fts_proto_alloc(budget, size); + if (!bytes->data) { + return -1; + } + + bytes->len = size; + memcpy(bytes->data, value->data.as_bytes, size); + return 0; + case PROTOBUF_C_TYPE_BOOL: + if (value->type != CFL_VARIANT_BOOL) { + return -1; + } + *(protobuf_c_boolean *) ptr = value->data.as_bool; + return 0; + case PROTOBUF_C_TYPE_ENUM: + case PROTOBUF_C_TYPE_INT32: + case PROTOBUF_C_TYPE_SINT32: + case PROTOBUF_C_TYPE_SFIXED32: + if (signed_integer(value, &signed_value) != 0 || + signed_value < INT32_MIN || signed_value > INT32_MAX) { + return -1; + } + *(int32_t *) ptr = (int32_t) signed_value; + return 0; + case PROTOBUF_C_TYPE_UINT32: + case PROTOBUF_C_TYPE_FIXED32: + if (unsigned_integer(value, &unsigned_value) != 0 || unsigned_value > UINT32_MAX) { + return -1; + } + *(uint32_t *) ptr = (uint32_t) unsigned_value; + return 0; + case PROTOBUF_C_TYPE_INT64: + case PROTOBUF_C_TYPE_SINT64: + case PROTOBUF_C_TYPE_SFIXED64: + return signed_integer(value, ptr); + case PROTOBUF_C_TYPE_UINT64: + case PROTOBUF_C_TYPE_FIXED64: + return unsigned_integer(value, ptr); + case PROTOBUF_C_TYPE_DOUBLE: + if (value->type == CFL_VARIANT_DOUBLE) { + *(double *) ptr = value->data.as_double; + } + else { + return -1; + } + return 0; + default: + return -1; + } +} + +static int write_repeated(const ProtobufCFieldDescriptor *field, struct cfl_variant *value, + ProtobufCMessage *message, struct fts_proto_budget *budget, int depth) +{ + size_t count; + size_t i; + size_t stride; + char *items; + struct cfl_list *head; + struct cfl_kvpair *pair; + Opentelemetry__Proto__Common__V1__KeyValue *entry; + ProtobufCMessage **messages; + + if (is_key_value(field)) { + if (value->type != CFL_VARIANT_KVLIST) { + return -1; + } + count = cfl_list_size(&value->data.as_kvlist->list); + } + else { + if (value->type != CFL_VARIANT_ARRAY) { + return -1; + } + count = value->data.as_array->entry_count; + } + + stride = scalar_size(field->type); + if (count > SIZE_MAX / stride) { + return -1; + } + + items = fts_proto_alloc(budget, count * stride); + if (!items) { + return -1; + } + + *(void **) ((char *) message + field->offset) = items; + *(size_t *) ((char *) message + field->quantifier_offset) = count; + + if (is_key_value(field)) { + i = 0; + messages = (ProtobufCMessage **) items; + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + entry = fts_proto_alloc(budget, sizeof(*entry)); + if (!entry) { + return -1; + } + + opentelemetry__proto__common__v1__key_value__init(entry); + messages[i++] = &entry->base; + + if (memchr(pair->key, 0, cfl_sds_len(pair->key)) || + !fts_valid_utf8(pair->key, cfl_sds_len(pair->key)) || + reserve(budget, cfl_sds_len(pair->key) + 1) != 0) { + return -1; + } + + entry->key = flb_strdup(pair->key); + entry->value = (void *) fts_proto_build( + &opentelemetry__proto__common__v1__any_value__descriptor, + pair->val, budget, depth + 1); + if (!entry->key) { + flb_errno(); + return -1; + } + + if (!entry->value) { + return -1; + } + } + } + else { + for (i = 0; i < count; i++) { + if (write_scalar(field, value->data.as_array->entries[i], items + i * stride, + budget, depth) != 0) { + return -1; + } + } + } + + return 0; +} + +ProtobufCMessage *fts_proto_build(const ProtobufCMessageDescriptor *descriptor, + struct cfl_variant *value, + struct fts_proto_budget *budget, int depth) +{ + ProtobufCMessage *message; + const ProtobufCFieldDescriptor *field; + struct cfl_list *head; + struct cfl_kvpair *pair; + int any; + int wrapper; + const char *name; + uint32_t *oneof; + + if (depth > FTS_MAX_DEPTH) { + return NULL; + } + + /* An adapter may retain a failed original item as opaque transport data. + * This path is private staging; policies never receive opaque messages. */ + if (value->type == CFL_VARIANT_BYTES && + (strcmp(descriptor->short_name, "ResourceMetrics") == 0 || + strcmp(descriptor->short_name, "ResourceSpans") == 0)) { + return protobuf_c_message_unpack(descriptor, NULL, cfl_variant_size_get(value), + (uint8_t *) value->data.as_bytes); + } + + message = fts_proto_alloc(budget, descriptor->sizeof_message); + if (!message) { + return NULL; + } + + descriptor->message_init(message); + any = strcmp(descriptor->short_name, "AnyValue") == 0; + wrapper = strcmp(descriptor->short_name, "ArrayValue") == 0 || + strcmp(descriptor->short_name, "KeyValueList") == 0; + + if (any) { + switch (value->type) { + case CFL_VARIANT_NULL: + return message; + case CFL_VARIANT_STRING: + name = "string_value"; + break; + case CFL_VARIANT_BYTES: + name = "bytes_value"; + break; + case CFL_VARIANT_BOOL: + name = "bool_value"; + break; + case CFL_VARIANT_INT: + case CFL_VARIANT_UINT: + name = "int_value"; + break; + case CFL_VARIANT_DOUBLE: + name = "double_value"; + break; + case CFL_VARIANT_ARRAY: + name = "array_value"; + break; + case CFL_VARIANT_KVLIST: + name = "kvlist_value"; + break; + default: + goto failure; + } + + field = protobuf_c_message_descriptor_get_field_by_name(descriptor, name); + *(uint32_t *) ((char *) message + field->quantifier_offset) = field->id; + + if (write_scalar(field, value, (char *) message + field->offset, budget, depth) != 0) { + goto failure; + } + return message; + } + + if (wrapper) { + if (write_repeated(&descriptor->fields[0], value, message, budget, depth) != 0) { + goto failure; + } + return message; + } + + if (value->type != CFL_VARIANT_KVLIST) { + goto failure; + } + + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + field = protobuf_c_message_descriptor_get_field_by_name(descriptor, pair->key); + if (!field || memchr(pair->key, 0, cfl_sds_len(pair->key)) || + fts_find_pair(value, pair->key, cfl_sds_len(pair->key)) != pair) { + goto failure; + } + + if (field->flags & PROTOBUF_C_FIELD_FLAG_ONEOF) { + oneof = (uint32_t *) ((char *) message + field->quantifier_offset); + if (*oneof != 0) { + goto failure; + } + *oneof = field->id; + } + else if (field->label == PROTOBUF_C_LABEL_OPTIONAL && field->quantifier_offset) { + *(protobuf_c_boolean *) ((char *) message + field->quantifier_offset) = 1; + } + + if (field->label == PROTOBUF_C_LABEL_REPEATED) { + if (write_repeated(field, pair->val, message, budget, depth) != 0) { + goto failure; + } + } + else if (write_scalar(field, pair->val, (char *) message + field->offset, + budget, depth) != 0) { + goto failure; + } + } + + return message; + +failure: + protobuf_c_message_free_unpacked(message, NULL); + return NULL; +} diff --git a/plugins/processor_fts/fts_proto.h b/plugins/processor_fts/fts_proto.h new file mode 100644 index 00000000000..19324881df4 --- /dev/null +++ b/plugins/processor_fts/fts_proto.h @@ -0,0 +1,39 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FTS_PROTO_H +#define FLB_FTS_PROTO_H + +#include "fts.h" +#include + +struct fts_proto_budget { + size_t used; + size_t limit; +}; + +void *fts_proto_alloc(void *data, size_t size); +void fts_proto_free(void *data, void *ptr); +struct cfl_variant *fts_proto_read(const ProtobufCMessage *message, + struct fts_proto_budget *budget, int depth); +ProtobufCMessage *fts_proto_build(const ProtobufCMessageDescriptor *descriptor, + struct cfl_variant *value, + struct fts_proto_budget *budget, int depth); + +#endif diff --git a/plugins/processor_fts/fts_runtime.c b/plugins/processor_fts/fts_runtime.c new file mode 100644 index 00000000000..b0bdd69f44c --- /dev/null +++ b/plugins/processor_fts/fts_runtime.c @@ -0,0 +1,436 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "fts.h" + +/* A policy cannot catch budget exhaustion and continue an endless loop. */ +static int policy_pcall(lua_State *l) +{ + struct fts_runtime *ctx; + void *data; + int ret; + + luaL_checktype(l, 1, LUA_TFUNCTION); + ret = lua_pcall(l, lua_gettop(l) - 1, LUA_MULTRET, 0); + + lua_getallocf(l, &data); + ctx = data; + if (!ctx->jit && ctx->instructions_left <= 0) { + return luaL_error(l, "Lua instruction limit exceeded"); + } + + lua_pushboolean(l, ret == 0); + lua_insert(l, 1); + return lua_gettop(l); +} + +/* Enforce the count budget in interpreted Lua, including script initialization. */ +static void budget_hook(lua_State *l, lua_Debug *ar) +{ + struct fts_runtime *ctx; + void *data; + + lua_getallocf(l, &data); + ctx = data; + ctx->instructions_left -= 1000; + + if (!ctx->jit && ctx->instructions_left <= 0) { + luaL_error(l, "Lua instruction limit exceeded"); + } +} + +static void *bounded_allocator(void *data, void *ptr, size_t old_size, size_t new_size) +{ + struct fts_runtime *ctx = data; + void *result; + size_t used; + + if (!ptr) { + old_size = 0; + } + + if (ctx->lua_bytes >= old_size) { + used = ctx->lua_bytes - old_size; + } + else { + used = 0; + } + + if (new_size > old_size && + (new_size > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - new_size || + used > ctx->memory_limit - ctx->native_bytes - new_size)) { + return NULL; + } + + result = ctx->original_allocator(ctx->allocator_data, ptr, old_size, new_size); + if (result || new_size == 0) { + ctx->lua_bytes = used + new_size; + } + + return result; +} + +static int policy_collectgarbage(lua_State *l) +{ + const char *option; + int arguments; + + option = luaL_optstring(l, 1, "collect"); + if (strcmp(option, "collect") != 0 && strcmp(option, "count") != 0 && + strcmp(option, "step") != 0) { + return luaL_error(l, "FTS collectgarbage supports collect, count and step only"); + } + + arguments = lua_gettop(l); + lua_pushvalue(l, lua_upvalueindex(1)); + lua_insert(l, 1); + lua_call(l, arguments, LUA_MULTRET); + return lua_gettop(l); +} + +static int restricted_require(lua_State *l) +{ + const char *name; + size_t length; + + name = luaL_checklstring(l, 1, &length); + if (length != sizeof("fluentbit") - 1 || memcmp(name, "fluentbit", length) != 0) { + return luaL_error(l, "only the fluentbit module is available in FTS policies"); + } + + lua_pushvalue(l, lua_upvalueindex(1)); + return 1; +} + +void fts_runtime_destroy(struct fts_runtime *ctx) +{ + fts_undo_commit(ctx); + flb_free(ctx->undo); + ctx->undo = NULL; + ctx->undo_size = 0; + + if (ctx->vm) { + flb_luajit_destroy(ctx->vm); + ctx->vm = NULL; + } +} + +/* + * Helper code has a private copy of the standard environment. Policies cannot + * replace validation primitives or discover this environment through getfenv(). + */ +static void copy_table(lua_State *l, int index) +{ + int source; + int target; + + if (index < 0 && index > LUA_REGISTRYINDEX) { + source = lua_gettop(l) + index + 1; + } + else { + source = index; + } + + lua_newtable(l); + target = lua_gettop(l); + + lua_pushnil(l); + while (lua_next(l, source)) { + lua_pushvalue(l, -2); + lua_pushvalue(l, -2); + lua_rawset(l, target); + lua_pop(l, 1); + } +} + +int fts_load_helpers(lua_State *l, const char *code, size_t size, const char *name) +{ + struct fts_runtime *ctx; + void *data; + + lua_getallocf(l, &data); + ctx = data; + + if (luaL_loadbuffer(l, code, size, name) != 0) { + return -1; + } + + lua_rawgeti(l, LUA_REGISTRYINDEX, ctx->helpers_env_ref); + lua_setfenv(l, -2); + return lua_pcall(l, 0, 0, 0); +} + +static int initialize_policy(lua_State *l) +{ + struct fts_runtime *ctx; + const char *blocked[] = {"io", "os", "debug", "jit", "ffi", "package", "coroutine", + "module", "load", "loadfile", "loadstring", "dofile", + "newproxy", "xpcall", "getfenv", "setfenv", "print", NULL}; + int i; + int ret; + int environment; + + ctx = lua_touserdata(l, 1); + copy_table(l, LUA_GLOBALSINDEX); + environment = lua_gettop(l); + + /* tables used by validation must also be independent of policy globals */ + lua_getglobal(l, "math"); + copy_table(l, -1); + lua_setfield(l, environment, "math"); + lua_pop(l, 1); + + lua_getglobal(l, "string"); + copy_table(l, -1); + lua_setglobal(l, "string"); + lua_pop(l, 1); + + lua_getglobal(l, "table"); + copy_table(l, -1); + lua_setfield(l, environment, "table"); + lua_pop(l, 1); + + lua_pushliteral(l, ""); + lua_getmetatable(l, -1); + lua_pushliteral(l, "string"); + lua_setfield(l, -2, "__metatable"); + lua_pop(l, 2); + + ctx->helpers_env_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + if (fts_register_values(l) != 0) { + return luaL_error(l, "cannot register fluentbit module"); + } + + if (ctx->adapter->register_helpers(l) != 0) { + return luaL_error(l, "cannot register FTS signal helpers"); + } + + lua_getglobal(l, "package"); + lua_getfield(l, -1, "loaded"); + lua_getfield(l, -1, "fluentbit"); + lua_pushvalue(l, -1); + ctx->module_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + /* public helper overrides do not affect the private adapter module */ + copy_table(l, -1); + lua_pushcclosure(l, restricted_require, 1); + lua_setglobal(l, "require"); + + /* removing globals alone leaves Lua 5.1 module() able to recover loaded libraries */ + for (i = 0; blocked[i]; i++) { + lua_pushnil(l); + lua_setfield(l, -2, blocked[i]); + } + lua_pop(l, 3); + + for (i = 0; blocked[i]; i++) { + lua_pushnil(l); + lua_setglobal(l, blocked[i]); + } + + lua_pushcfunction(l, policy_pcall); + lua_setglobal(l, "pcall"); + + lua_getglobal(l, "collectgarbage"); + lua_pushcclosure(l, policy_collectgarbage, 1); + lua_setglobal(l, "collectgarbage"); + + if (ctx->code) { + ret = flb_luajit_load_buffer(ctx->vm, ctx->code, strlen(ctx->code), "fts_policy"); + } + else { + ret = flb_luajit_load_script(ctx->vm, ctx->script); + } + + if (ret != 0) { + return lua_error(l); + } + + lua_call(l, 0, 0); + + lua_getglobal(l, ctx->call); + if (!lua_isfunction(l, -1)) { + return luaL_error(l, "FTS callback function not found"); + } + ctx->policy_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + /* field-name strings are callback upvalues, see fts_getfield() */ + for (i = 0; i < FTS_KEY_COUNT; i++) { + lua_rawgeti(l, LUA_REGISTRYINDEX, ctx->key_refs[i]); + } + lua_pushcclosure(l, ctx->adapter->run_callback, FTS_KEY_COUNT); + ctx->callback_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + if (!lua_checkstack(l, 32)) { + return luaL_error(l, "cannot reserve policy stack"); + } + + return 0; +} + +int fts_runtime_initialize(struct fts_runtime *ctx, struct flb_config *config) +{ + lua_State *l; + int ret; + + ctx->vm = flb_luajit_create(config); + if (!ctx->vm) { + return FLB_PROCESSOR_FAILURE; + } + l = ctx->vm->state; + + ret = luaJIT_setmode(l, 0, LUAJIT_MODE_ENGINE | (ctx->jit ? LUAJIT_MODE_ON : LUAJIT_MODE_OFF)); + if (ctx->jit && !ret) { + lua_pushliteral(l, "JIT is unavailable in this LuaJIT build"); + return FLB_PROCESSOR_FAILURE; + } + + ctx->original_allocator = lua_getallocf(l, &ctx->allocator_data); + ctx->lua_bytes = (size_t) lua_gc(l, LUA_GCCOUNT, 0) * 1024 + lua_gc(l, LUA_GCCOUNTB, 0); + lua_setallocf(l, bounded_allocator, ctx); + + ctx->instructions_left = ctx->instruction_limit; + if (!ctx->jit) { + lua_sethook(l, budget_hook, LUA_MASKCOUNT, 1000); + } + + lua_pushcfunction(l, initialize_policy); + lua_pushlightuserdata(l, ctx); + ret = lua_pcall(l, 1, 0, 0); + if (ret != 0) { + return FLB_PROCESSOR_FAILURE; + } + + return FLB_PROCESSOR_SUCCESS; +} + +/* + * Native values are invisible to the Lua allocator, so the collector is paced + * explicitly: each chunk performs incremental work proportional to the native + * bytes it charged. A full collection only runs on emergency or when close to + * the memory limit, avoiding a full cycle per chunk or error. + */ +void fts_collect(struct fts_runtime *ctx, int emergency) +{ + lua_State *l = ctx->vm->state; + size_t used = ctx->native_bytes + ctx->lua_bytes; + size_t allocated; + + allocated = ctx->native_allocated; + ctx->native_allocated = 0; + + if (emergency || used > ctx->memory_limit - ctx->memory_limit / 4) { + lua_gc(l, LUA_GCCOLLECT, 0); + } + else if (allocated > 0) { + lua_gc(l, LUA_GCSTEP, (int) (allocated / 1024 < INT_MAX ? allocated / 1024 + 1 : INT_MAX)); + } + else if (used > ctx->memory_limit / 2) { + lua_gc(l, LUA_GCSTEP, 64); + } +} + +/* adapters supply their own snapshot and result contract inside the protected call */ +int fts_runtime_call(struct fts_runtime *ctx, void *data, int results) +{ + lua_State *l; + int ret; + int base; + + l = ctx->vm->state; + base = lua_gettop(l); + ctx->instructions_left = ctx->instruction_limit; + + lua_rawgeti(l, LUA_REGISTRYINDEX, ctx->callback_ref); + lua_pushlightuserdata(l, data); + ret = lua_pcall(l, 1, results, 0); + if (ret == 0 && !ctx->jit && ctx->instructions_left <= 0) { + lua_settop(l, base); + lua_pushliteral(l, "Lua instruction limit exceeded"); + ret = LUA_ERRRUN; + } + + if (ret != 0) { + /* unwound snapshots and garbage must not poison subsequent callbacks */ + fts_collect(ctx, ret == LUA_ERRMEM); + if (!lua_gc(l, LUA_GCISRUNNING, 0)) { + lua_gc(l, LUA_GCRESTART, -1); + } + } + + return ret; +} + +void fts_report_policy_error(struct flb_processor_instance *ins) +{ + struct fts_runtime *ctx; + struct flb_processor *processor; + struct flb_input_instance *input; + struct flb_output_instance *output; + char stage[32]; + char *labels[5]; + + if (!ins->pu || !ins->pu->parent) { + return; + } + + processor = ins->pu->parent; + if (!processor->cmt_errors) { + return; + } + + labels[0] = "unknown"; + labels[1] = "unknown"; + if (processor->source_plugin_type == FLB_PLUGIN_INPUT && processor->data) { + input = processor->data; + labels[0] = "input"; + labels[1] = (char *) flb_input_name(input); + } + else if (processor->source_plugin_type == FLB_PLUGIN_OUTPUT && processor->data) { + output = processor->data; + labels[0] = "output"; + labels[1] = (char *) flb_output_name(output); + } + + snprintf(stage, sizeof(stage), "%zu", ins->pu->stage); + labels[2] = ins->pu->name; + labels[3] = stage; + + ctx = ins->context; + if (ctx->adapter->event_type == FLB_PROCESSOR_METRICS) { + labels[4] = "metrics"; + } + else if (ctx->adapter->event_type == FLB_PROCESSOR_TRACES) { + labels[4] = "traces"; + } + else { + labels[4] = "logs"; + } + + cmt_counter_inc(processor->cmt_errors, cfl_time_now(), 5, labels); +} + diff --git a/plugins/processor_fts/fts_telemetry.c b/plugins/processor_fts/fts_telemetry.c new file mode 100644 index 00000000000..dfa9a1ba6bd --- /dev/null +++ b/plugins/processor_fts/fts_telemetry.c @@ -0,0 +1,815 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include "fts_telemetry.h" +#include "fts_helpers.h" + +struct telemetry_call { + struct fts_runtime *ctx; + struct cfl_variant *snapshot; + const char *tag; + int tag_len; +}; + +static struct cfl_variant *field(struct cfl_variant *map, const char *key) +{ + struct cfl_kvpair *pair; + + if (!map || map->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + pair = fts_find_pair(map, key, strlen(key)); + if (!pair) { + return NULL; + } + + return pair->val; +} + +static struct cfl_variant *new_map(void) +{ + struct cfl_kvlist *map; + struct cfl_variant *value; + + map = cfl_kvlist_create(); + if (!map) { + return NULL; + } + + value = cfl_variant_create_from_kvlist(map); + if (!value) { + cfl_kvlist_destroy(map); + } + + return value; +} + +static struct cfl_variant *new_array(void) +{ + struct cfl_array *array; + struct cfl_variant *value; + + array = cfl_array_create(0); + if (!array) { + return NULL; + } + cfl_array_resizable(array, CFL_TRUE); + + value = cfl_variant_create_from_array(array); + if (!value) { + cfl_array_destroy(array); + } + + return value; +} + +/* the receiver owns value on success and failure */ +static int put(struct cfl_variant *map, const char *key, struct cfl_variant *value) +{ + if (!value || cfl_kvlist_insert(map->data.as_kvlist, (char *) key, value) != 0) { + cfl_variant_destroy(value); + return -1; + } + + return 0; +} + +static int clone_field(struct cfl_variant *map, const char *name, struct cfl_variant *source) +{ + struct cfl_variant *value; + + if (source) { + value = fts_copy_value(source, 0); + } + else { + value = new_map(); + } + + return put(map, name, value); +} + +static struct cfl_variant *snapshot(struct cfl_variant *resource, struct cfl_variant *scope, + struct cfl_variant *item) +{ + struct cfl_variant *result; + struct cfl_variant *context; + struct cfl_variant *url; + struct cfl_variant *value; + + result = new_map(); + if (!result) { + return NULL; + } + + if (clone_field(result, "item", item) != 0 || + clone_field(result, "resource", field(resource, "resource")) != 0 || + clone_field(result, "scope", field(scope, "scope")) != 0) { + goto failure; + } + + context = field(result, "resource"); + if (!field(context, "attributes") && put(context, "attributes", new_map()) != 0) { + goto failure; + } + + context = field(result, "scope"); + if (!field(context, "attributes") && put(context, "attributes", new_map()) != 0) { + goto failure; + } + + url = field(resource, "schema_url"); + if (url) { + value = fts_copy_value(url, 0); + } + else { + value = cfl_variant_create_from_string(""); + } + if (put(result, "resource_schema_url", value) != 0) { + goto failure; + } + + url = field(scope, "schema_url"); + if (url) { + value = fts_copy_value(url, 0); + } + else { + value = cfl_variant_create_from_string(""); + } + if (put(result, "scope_schema_url", value) != 0) { + goto failure; + } + + return result; + +failure: + cfl_variant_destroy(result); + return NULL; +} + +int fts_register_telemetry_helpers(lua_State *l) +{ + if (fts_load_helpers(l, fts_telemetry_helpers, sizeof(fts_telemetry_helpers) - 1, + "fluentbit.telemetry_helpers") != 0) { + return -1; + } + + return 0; +} + +int fts_run_telemetry(lua_State *l) +{ + struct telemetry_call *call; + struct fts_value *owner; + struct fts_value *emissions; + const char *signal; + const char *result; + size_t length; + size_t i; + int original; + int raw; + int output; + int module; + int view; + int drop = FLB_FALSE; + + call = lua_touserdata(l, 1); + if (call->ctx->adapter->event_type == FLB_PROCESSOR_METRICS) { + signal = "metrics"; + } + else { + signal = "traces"; + } + + /* mutable snapshot handed to the policy */ + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, fts_value_cost(call->snapshot, 0)); + owner->owned = fts_copy_value(call->snapshot, 0); + if (!owner->owned) { + return luaL_error(l, "cannot create telemetry snapshot"); + } + raw = lua_gettop(l); + + /* read-only copy used to validate the edits */ + owner = fts_push_owned(l, NULL, FLB_TRUE); + fts_charge_value(l, owner, fts_value_cost(call->snapshot, 0)); + owner->owned = fts_copy_value(call->snapshot, 0); + if (!owner->owned) { + return luaL_error(l, "cannot preserve original telemetry fields"); + } + original = lua_gettop(l); + + emissions = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, emissions, 256); + emissions->owned = new_array(); + if (!emissions->owned) { + return luaL_error(l, "cannot create emissions array"); + } + output = lua_gettop(l); + + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->module_ref); + module = lua_gettop(l); + + lua_getfield(l, module, "_fts_wrap"); + lua_pushvalue(l, raw); + lua_pushstring(l, signal); + lua_pushvalue(l, output); + if (call->tag) { + lua_pushlstring(l, call->tag, call->tag_len); + } + else { + lua_pushlstring(l, "", 0); + } + lua_call(l, 4, 1); + view = lua_gettop(l); + + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->policy_ref); + lua_pushvalue(l, view); + lua_call(l, 1, 1); + + if (!lua_isnil(l, -1)) { + result = NULL; + if (lua_type(l, -1) == LUA_TSTRING) { + result = lua_tolstring(l, -1, &length); + } + + if (!result || length != 4 || (memcmp(result, "keep", 4) && memcmp(result, "drop", 4))) { + return luaL_error(l, "callback must return nil, fb.KEEP or fb.DROP"); + } + drop = memcmp(result, "drop", 4) == 0; + } + + if (!drop) { + lua_getfield(l, module, "_fts_validate_telemetry"); + lua_pushvalue(l, raw); + lua_pushstring(l, signal); + lua_pushvalue(l, original); + lua_call(l, 3, 0); + } + + for (i = 0; i < emissions->owned->data.as_array->entry_count; i++) { + lua_getfield(l, module, "_fts_validate_telemetry"); + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, fts_value_cost(emissions->owned->data.as_array->entries[i], 0)); + owner->owned = fts_copy_value(emissions->owned->data.as_array->entries[i], 0); + if (!owner->owned) { + return luaL_error(l, "cannot validate emitted metric"); + } + lua_pushstring(l, signal); + lua_call(l, 2, 0); + } + + lua_pushstring(l, drop ? "drop" : "keep"); + lua_pushvalue(l, raw); + lua_pushvalue(l, output); + return 3; +} + +static struct cfl_variant *group_from_snapshot(struct cfl_variant *value, int metrics) +{ + struct cfl_variant *resource; + struct cfl_variant *scope; + struct cfl_variant *scopes; + struct cfl_variant *items; + struct cfl_variant *item = NULL; + + resource = new_map(); + scope = new_map(); + scopes = new_array(); + items = new_array(); + if (!resource || !scope || !scopes || !items) { + goto failure; + } + + if (clone_field(resource, "resource", field(value, "resource")) != 0 || + clone_field(resource, "schema_url", field(value, "resource_schema_url")) != 0 || + clone_field(scope, "scope", field(value, "scope")) != 0 || + clone_field(scope, "schema_url", field(value, "scope_schema_url")) != 0) { + goto failure; + } + + item = fts_copy_value(field(value, "item"), 0); + if (!item || cfl_array_append(items->data.as_array, item) != 0) { + goto failure; + } + item = NULL; + + if (put(scope, metrics ? "metrics" : "spans", items) != 0) { + items = NULL; + goto failure; + } + items = NULL; + + if (cfl_array_append(scopes->data.as_array, scope) != 0) { + goto failure; + } + scope = NULL; + + if (put(resource, metrics ? "scope_metrics" : "scope_spans", scopes) != 0) { + scopes = NULL; + goto failure; + } + + return resource; + +failure: + cfl_variant_destroy(resource); + cfl_variant_destroy(scope); + cfl_variant_destroy(scopes); + cfl_variant_destroy(items); + cfl_variant_destroy(item); + return NULL; +} + +static int stage(struct cfl_variant *groups, struct cfl_variant *value, int metrics, + size_t *used, size_t limit) +{ + struct cfl_variant *group; + ProtobufCMessage *message; + struct fts_proto_budget budget = {0, limit}; + const ProtobufCMessageDescriptor *descriptor; + ProtobufCMessage *item; + size_t cost; + size_t length; + size_t prefix; + + group = group_from_snapshot(value, metrics); + if (!group) { + return -1; + } + + if (metrics) { + descriptor = &opentelemetry__proto__metrics__v1__resource_metrics__descriptor; + } + else { + descriptor = &opentelemetry__proto__trace__v1__resource_spans__descriptor; + } + + message = fts_proto_build(descriptor, group, &budget, 0); + if (!message) { + cfl_variant_destroy(group); + return -1; + } + + if (metrics) { + item = &((Opentelemetry__Proto__Metrics__V1__ResourceMetrics *) message)-> + scope_metrics[0]->metrics[0]->base; + } + else { + item = &((Opentelemetry__Proto__Trace__V1__ResourceSpans *) message)-> + scope_spans[0]->spans[0]->base; + } + + /* + * Account item bytes and its repeated-field framing here. Coalesced + * resource/scope bytes are checked once in the final serialized output. + */ + cost = protobuf_c_message_get_packed_size(item); + prefix = 1; + for (length = cost; length >= 128; length >>= 7) { + prefix++; + } + cost += prefix + 1; + protobuf_c_message_free_unpacked(message, NULL); + + if (cost > limit || *used > limit - cost || + cfl_array_append(groups->data.as_array, group) != 0) { + cfl_variant_destroy(group); + return -1; + } + + *used += cost; + return 0; +} + +/* Coalesce adjacent unchanged contexts without teaching policies about groups. */ +static int append_group(struct cfl_variant *groups, struct cfl_variant *group, int metrics) +{ + struct cfl_array *list; + struct cfl_variant *previous; + struct cfl_variant *a; + struct cfl_variant *b; + struct cfl_variant *items; + struct cfl_variant *source; + struct cfl_variant *item; + const char *scope_key = metrics ? "scope_metrics" : "scope_spans"; + const char *item_key = metrics ? "metrics" : "spans"; + size_t i; + + list = groups->data.as_array; + if (group->type != CFL_VARIANT_BYTES && list->entry_count > 0 && + list->entries[list->entry_count - 1]->type != CFL_VARIANT_BYTES) { + previous = list->entries[list->entry_count - 1]; + a = field(previous, scope_key)->data.as_array->entries[0]; + b = field(group, scope_key)->data.as_array->entries[0]; + + if (fts_values_equal(field(previous, "resource"), field(group, "resource"), 0) && + fts_values_equal(field(previous, "schema_url"), field(group, "schema_url"), 0) && + fts_values_equal(field(a, "scope"), field(b, "scope"), 0) && + fts_values_equal(field(a, "schema_url"), field(b, "schema_url"), 0)) { + items = field(a, item_key); + source = field(b, item_key); + for (i = 0; i < source->data.as_array->entry_count; i++) { + item = source->data.as_array->entries[i]; + source->data.as_array->entries[i] = NULL; + cfl_container_release_variant(item); + if (cfl_array_append(items->data.as_array, item) != 0) { + cfl_variant_destroy(item); + cfl_variant_destroy(group); + return -1; + } + } + cfl_variant_destroy(group); + return 0; + } + } + + if (cfl_array_append(groups->data.as_array, group) != 0) { + cfl_variant_destroy(group); + return -1; + } + + return 0; +} + +/* Decode resource/scope context without recursively materializing their items. */ +static struct cfl_variant *read_context(const ProtobufCMessage *message, + const char *children, struct fts_proto_budget *budget) +{ + const ProtobufCFieldDescriptor *descriptor; + ProtobufCMessage *copy; + struct cfl_variant *result; + + descriptor = protobuf_c_message_descriptor_get_field_by_name(message->descriptor, children); + copy = flb_malloc(message->descriptor->sizeof_message); + if (!copy) { + flb_errno(); + return NULL; + } + + if (!descriptor) { + flb_free(copy); + return NULL; + } + + /* shallow copy with the children field emptied */ + memcpy(copy, message, message->descriptor->sizeof_message); + *(size_t *) ((char *) copy + descriptor->quantifier_offset) = 0; + *(void **) ((char *) copy + descriptor->offset) = NULL; + + budget->used = 0; + result = fts_proto_read(copy, budget, 0); + flb_free(copy); + + return result; +} + +static ProtobufCMessage **children_of(const ProtobufCMessage *message, + const char *name, size_t *count) +{ + const ProtobufCFieldDescriptor *field; + + field = protobuf_c_message_descriptor_get_field_by_name(message->descriptor, name); + *count = *(size_t *) ((char *) message + field->quantifier_offset); + return *(ProtobufCMessage ***) ((char *) message + field->offset); +} + +/* + * Keep a failed original item without requiring it to fit the typed API's + * nesting/memory budget. Its siblings still receive independent callbacks. + */ +static struct cfl_variant *opaque_group(ProtobufCMessage *resource, + ProtobufCMessage *scope, ProtobufCMessage *item, + int metrics) +{ + const ProtobufCFieldDescriptor *resource_field; + const ProtobufCFieldDescriptor *scope_field; + const char *scope_name; + const char *item_name; + ProtobufCMessage *resource_copy; + ProtobufCMessage *scope_copy; + struct cfl_variant *result; + uint8_t *buffer; + size_t size; + + resource_copy = flb_malloc(resource->descriptor->sizeof_message); + scope_copy = flb_malloc(scope->descriptor->sizeof_message); + if (!resource_copy || !scope_copy) { + flb_errno(); + flb_free(resource_copy); + flb_free(scope_copy); + return NULL; + } + + /* shallow copies linked so they only reference the single item */ + memcpy(resource_copy, resource, resource->descriptor->sizeof_message); + memcpy(scope_copy, scope, scope->descriptor->sizeof_message); + + if (metrics) { + scope_name = "scope_metrics"; + item_name = "metrics"; + } + else { + scope_name = "scope_spans"; + item_name = "spans"; + } + + resource_field = protobuf_c_message_descriptor_get_field_by_name(resource->descriptor, + scope_name); + scope_field = protobuf_c_message_descriptor_get_field_by_name(scope->descriptor, item_name); + + *(size_t *) ((char *) resource_copy + resource_field->quantifier_offset) = 1; + *(ProtobufCMessage ***) ((char *) resource_copy + resource_field->offset) = &scope_copy; + *(size_t *) ((char *) scope_copy + scope_field->quantifier_offset) = 1; + *(ProtobufCMessage ***) ((char *) scope_copy + scope_field->offset) = &item; + + size = protobuf_c_message_get_packed_size(resource_copy); + buffer = flb_malloc(size ? size : 1); + if (!buffer) { + flb_errno(); + flb_free(scope_copy); + flb_free(resource_copy); + return NULL; + } + + protobuf_c_message_pack(resource_copy, buffer); + result = cfl_variant_create_from_bytes((char *) buffer, size, CFL_FALSE); + flb_free(buffer); + + flb_free(scope_copy); + flb_free(resource_copy); + + return result; +} + +int fts_telemetry_transform(struct flb_processor_instance *ins, char *buffer, size_t size, + char **output, size_t *output_size, const char *tag, int tag_len) +{ + struct fts_runtime *ctx = ins->context; + struct fts_proto_budget budget; + ProtobufCMessage *message = NULL; + const ProtobufCMessageDescriptor *descriptor; + struct cfl_variant *resource = NULL; + struct cfl_variant *scope = NULL; + struct cfl_variant *original_item = NULL; + struct cfl_variant *result = NULL; + struct cfl_variant *groups = NULL; + ProtobufCMessage **resources; + ProtobufCMessage **scopes; + ProtobufCMessage **items; + size_t resource_count; + size_t scope_count; + size_t item_count; + struct cfl_variant *value = NULL; + struct cfl_variant *pending = NULL; + struct cfl_variant *item; + struct fts_value *edited; + struct fts_value *emissions; + struct telemetry_call call; + lua_State *l = ctx->vm->state; + size_t r; + size_t s; + size_t i; + size_t e; + size_t used = 0; + size_t pending_cost; + int metrics = ctx->adapter->event_type == FLB_PROCESSOR_METRICS; + const char *resource_key = metrics ? "resource_metrics" : "resource_spans"; + const char *scope_key = metrics ? "scope_metrics" : "scope_spans"; + const char *item_key = metrics ? "metrics" : "spans"; + int base; + int ret; + int errors = 0; + int failed; + const char *phase = "decode OTLP"; + + *output = NULL; + *output_size = 0; + + if (metrics) { + descriptor = &opentelemetry__proto__metrics__v1__metrics_data__descriptor; + } + else { + descriptor = + &opentelemetry__proto__collector__trace__v1__export_trace_service_request__descriptor; + } + + budget.used = 0; + budget.limit = ctx->memory_limit; + + message = protobuf_c_message_unpack(descriptor, NULL, size, (uint8_t *) buffer); + if (!message) { + goto failure; + } + + result = new_map(); + groups = new_array(); + if (!result || !groups) { + goto failure; + } + + phase = "read resource groups"; + resources = children_of(message, resource_key, &resource_count); + for (r = 0; r < resource_count; r++) { + resource = read_context(resources[r], scope_key, &budget); + scopes = children_of(resources[r], scope_key, &scope_count); + for (s = 0; s < scope_count; s++) { + scope = read_context(scopes[s], item_key, &budget); + items = children_of(scopes[s], item_key, &item_count); + for (i = 0; i < item_count; i++) { + phase = "create telemetry snapshot"; + budget.used = 0; + + original_item = NULL; + if (resource && scope) { + original_item = fts_proto_read(items[i], &budget, 0); + } + + value = NULL; + if (original_item) { + value = snapshot(resource, scope, original_item); + } + cfl_variant_destroy(original_item); + original_item = NULL; + + if (!value) { + errors++; + if (errors <= 5) { + flb_plg_error(ins, "FTS telemetry item exceeds snapshot limits; %s", + strcmp(ctx->error_mode, "keep") == 0 ? + "original item kept" : "item dropped"); + } + + if (strcmp(ctx->error_mode, "keep") == 0) { + item = opaque_group(resources[r], scopes[s], items[i], metrics); + if (!item || append_group(groups, item, metrics) != 0) { + goto failure; + } + } + continue; + } + + base = lua_gettop(l); + call.ctx = ctx; + call.snapshot = value; + call.tag = tag; + call.tag_len = tag_len; + ret = fts_runtime_call(ctx, &call, 3); + + pending = new_array(); + pending_cost = used; + failed = ret != 0 || !pending; + + if (!failed) { + edited = lua_touserdata(l, base + 2); + emissions = lua_touserdata(l, base + 3); + + if (strcmp(lua_tostring(l, base + 1), "keep") == 0 && + stage(pending, edited->owned, metrics, &pending_cost, + ctx->output_limit) != 0) { + failed = FLB_TRUE; + } + + for (e = 0; !failed && e < emissions->owned->data.as_array->entry_count; e++) { + if (stage(pending, emissions->owned->data.as_array->entries[e], metrics, + &pending_cost, ctx->output_limit) != 0) { + failed = FLB_TRUE; + } + } + } + + if (failed) { + errors++; + if (errors <= 5) { + flb_plg_error(ins, "FTS telemetry policy failed: %.256s; edits rolled back", + ret && lua_type(l, -1) == LUA_TSTRING ? + lua_tostring(l, -1) : + "invalid telemetry fields or output limit"); + } + + /* restart staging, keeping the original item when configured */ + cfl_variant_destroy(pending); + pending = new_array(); + pending_cost = used; + if (!pending || (strcmp(ctx->error_mode, "keep") == 0 && + stage(pending, value, metrics, &pending_cost, + ctx->output_limit) != 0)) { + lua_settop(l, base); + goto failure; + } + } + + for (e = 0; e < pending->data.as_array->entry_count; e++) { + item = pending->data.as_array->entries[e]; + pending->data.as_array->entries[e] = NULL; + cfl_container_release_variant(item); + if (append_group(groups, item, metrics) != 0) { + lua_settop(l, base); + goto failure; + } + } + + used = pending_cost; + cfl_variant_destroy(pending); + pending = NULL; + cfl_variant_destroy(value); + value = NULL; + lua_settop(l, base); + } + cfl_variant_destroy(scope); + scope = NULL; + } + cfl_variant_destroy(resource); + resource = NULL; + } + + if (put(result, resource_key, groups) != 0) { + groups = NULL; + goto failure; + } + groups = NULL; + + protobuf_c_message_free_unpacked(message, NULL); + message = NULL; + + /* transport staging is bounded by output_limit, independently of the VM */ + budget.used = 0; + budget.limit = SIZE_MAX; + phase = "encode typed OTLP values"; + + message = fts_proto_build(descriptor, result, &budget, 0); + if (!message) { + goto failure; + } + + *output_size = protobuf_c_message_get_packed_size(message); + if (*output_size > ctx->output_limit) { + goto failure; + } + + *output = flb_malloc(*output_size ? *output_size : 1); + if (!*output) { + flb_errno(); + goto failure; + } + + protobuf_c_message_pack(message, (uint8_t *) *output); + protobuf_c_message_free_unpacked(message, NULL); + + cfl_variant_destroy(resource); + cfl_variant_destroy(scope); + cfl_variant_destroy(original_item); + cfl_variant_destroy(result); + + ctx->instructions_left = ctx->instruction_limit; + lua_gc(l, LUA_GCCOLLECT, 0); + + if (errors > 5) { + flb_plg_warn(ins, "%d additional telemetry policy errors in this chunk", errors - 5); + } + + return errors; + +failure: + if (message) { + protobuf_c_message_free_unpacked(message, NULL); + } + + cfl_variant_destroy(resource); + cfl_variant_destroy(scope); + cfl_variant_destroy(original_item); + cfl_variant_destroy(result); + cfl_variant_destroy(groups); + cfl_variant_destroy(value); + cfl_variant_destroy(pending); + + ctx->instructions_left = ctx->instruction_limit; + lua_gc(l, LUA_GCCOLLECT, 0); + + flb_free(*output); + *output = NULL; + *output_size = 0; + + flb_plg_error(ins, "cannot construct FTS telemetry output (%s); edits discarded", phase); + return -1; +} diff --git a/plugins/processor_fts/fts_telemetry.h b/plugins/processor_fts/fts_telemetry.h new file mode 100644 index 00000000000..425b4131c62 --- /dev/null +++ b/plugins/processor_fts/fts_telemetry.h @@ -0,0 +1,40 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FTS_TELEMETRY_H +#define FLB_FTS_TELEMETRY_H + +#include "fts_proto.h" + +extern const struct fts_signal_adapter fts_metrics_adapter; +extern const struct fts_signal_adapter fts_traces_adapter; + +/* shared metrics and traces policy path */ +int fts_register_telemetry_helpers(lua_State *l); +int fts_run_telemetry(lua_State *l); +int fts_telemetry_transform(struct flb_processor_instance *ins, char *buffer, size_t size, + char **output, size_t *output_size, const char *tag, int tag_len); + +/* signal entry points */ +int fts_process_metrics(struct flb_processor_instance *ins, struct cmt *input, + struct cmt **output, const char *tag, int tag_len); +int fts_process_traces(struct flb_processor_instance *ins, struct ctrace *input, + struct ctrace **output, const char *tag, int tag_len); + +#endif diff --git a/plugins/processor_fts/fts_telemetry_helpers.lua b/plugins/processor_fts/fts_telemetry_helpers.lua new file mode 100644 index 00000000000..0cad262021b --- /dev/null +++ b/plugins/processor_fts/fts_telemetry_helpers.lua @@ -0,0 +1,571 @@ +local fb = package.loaded.fluentbit +local kinds = {'gauge', 'sum', 'histogram', 'exponential_histogram', 'summary'} +local integer_properties = {kind=true, flags=true, dropped_attributes_count=true, + dropped_events_count=true, dropped_links_count=true} + +local function hex(bytes) + return (bytes:gsub('.', function(c) return string.format('%02x', c:byte()) end)) +end + +local function unhex(text, size, optional) + assert(type(text) == 'string', 'ID must be a hexadecimal string') + if optional and text == '' then return fb.bytes('') end + assert(#text == size * 2 and text:match('^[%x]+$') and not text:match('^0+$'), 'invalid ID') + return fb.bytes((text:gsub('..', function(c) return string.char(tonumber(c, 16)) end))) +end + +local function object(raw) + return setmetatable({}, { + __index=function(_, key) + local value = raw:get(key) + if not value then return nil end + local kind = value:type() + if key == 'trace_id' or key == 'span_id' or key == 'parent_span_id' then + return hex(value:value()) + end + if kind == 'string' or kind == 'bool' or kind == 'double' then return value:value() end + if integer_properties[key] then return value:number() end + return value + end, + __newindex=function(_, key, value) + if key == 'trace_id' or key == 'span_id' or key == 'parent_span_id' then + value = unhex(value, key == 'trace_id' and 16 or 8, key == 'parent_span_id') + elseif integer_properties[key] and type(value) == 'number' then + value = fb.int64(value) + end + raw:set(key, value) + end, + __metatable='FTS telemetry object' + }) +end + +local roots = setmetatable({}, {__mode='k'}) + +local function metric_data(item) + for _, kind in ipairs(kinds) do + local data = item:get(kind) + if data then return kind, data end + end + error('metric has no supported data type') +end + +function fb._fts_wrap(root, signal, emissions, tag) + local item = root:get('item') + local view = object(item) + local function context(key) + local fields = object(root:get(key)) + return setmetatable({}, { + __index=function(_, name) + if name == 'schema_url' then return root:get_string(key .. '_schema_url') end + return fields[name] + end, + __newindex=function(_, name, value) + assert(name ~= 'attributes', 'change context attributes through set/remove') + if name == 'schema_url' then root:set(key .. '_schema_url', value) + else fields[name] = value end + end, + __metatable='FTS context' + }) + end + local resource, scope = context('resource'), context('scope') + local proxy = setmetatable({}, { + __index=function(_, key) + if key == 'signal' then return signal end + if key == 'tag' then return tag end + if key == 'resource' then return resource end + if key == 'scope' then return scope end + if signal == 'metrics' then + local kind, data = metric_data(item) + if key == 'type' then return kind end + if key == 'datapoints' then return data:get('data_points') end + if key == 'aggregation_temporality' then + local value = data:get(key) + return value and ({[0]='unspecified',[1]='delta',[2]='cumulative'})[value:number()] + end + if key == 'is_monotonic' then + local value = data:get(key); return value and value:value() + end + if key == 'emit' then + return function() + assert(emissions, 'metric emission is unavailable') + emissions:append(root) + return fb._fts_wrap(emissions:get(emissions:len()), signal, nil, tag) + end + end + end + return view[key] + end, + __newindex=function(_, key, value) + assert(key ~= 'resource' and key ~= 'scope' and key ~= 'signal' and key ~= 'tag', + 'change context fields through their existing views') + if signal == 'metrics' then + local kind, data = metric_data(item) + if key == 'datapoints' then data:set('data_points', value); return end + if key == 'aggregation_temporality' then + local number = ({unspecified=0,delta=1,cumulative=2})[value] + assert(number and kind ~= 'gauge' and kind ~= 'summary', 'invalid temporality') + data:set(key, fb.int64(number)); return + end + if key == 'is_monotonic' then + assert(kind == 'sum' and type(value) == 'boolean', 'monotonicity requires a sum') + data:set(key, value); return + end + if key == 'type' then + assert((kind == 'gauge' or kind == 'sum') and (value == 'gauge' or value == 'sum'), + 'only gauge/sum conversion preserves the datapoint model') + if value == kind then return end + local points = data:get('data_points') + local converted = fb.map(); converted:set('data_points', points) + if value == 'sum' then + converted:set('aggregation_temporality', fb.int64(2)) + converted:set('is_monotonic', false) + end + item:remove(kind); item:set(value, converted); return + end + end + view[key] = value + end, + __metatable='FTS telemetry record' + }) + roots[proxy] = root + return proxy +end + +function fb.metric_value(point) + return point:get('as_int') or point:get('as_double') +end + +function fb.set_metric_value(point, value) + if type(value) == 'number' then + assert(value == value and value ~= math.huge and value ~= -math.huge, 'nonfinite metric edit') + point:remove('as_int'); point:set('as_double', value) + else + local kind = value:type() + assert(kind == 'int64' or kind == 'uint64' or kind == 'double', 'metric value must be numeric') + if kind == 'double' then point:remove('as_int'); point:set('as_double', value) + else point:remove('as_double'); point:set('as_int', value) end + end +end + +function fb.copy_metric(metric, name, description, unit) + local copy = metric:emit() + if name then copy.name = name end + if description then copy.description = description end + if unit then copy.unit = unit end + return copy +end + +function fb.scale_metric(metric, factor, unit) + assert(type(factor) == 'number' and factor > 0 and factor < math.huge, 'scale must be finite and positive') + assert(metric.type ~= 'exponential_histogram', 'exponential histogram scaling requires bucket remapping') + for i=1,metric.datapoints:len() do + local point = metric.datapoints:get(i) + local value = fb.metric_value(point) + if value then + local number = value:number() * factor + if value:type() == 'int64' and factor == math.floor(factor) then + fb.set_metric_value(point, fb.int64(number)) + else fb.set_metric_value(point, number) end + else + for _, key in ipairs({'sum','min','max'}) do + local number = point:get_number(key) + if number then point:set(key, number * factor) end + end + local bounds = point:get('explicit_bounds') + if bounds then + for j=1,bounds:len() do bounds:set(j, bounds:get(j):number() * factor) end + end + local quantiles = point:get('quantile_values') + if quantiles then + for j=1,quantiles:len() do + local quantile = quantiles:get(j) + quantile:set('value', quantile:get_number('value') * factor) + end + end + end + local exemplars = point:get('exemplars') + if exemplars then + for j=1,exemplars:len() do + local exemplar = exemplars:get(j) + local original = fb.metric_value(exemplar) + if original then + local scaled = original:number() * factor + if original:type() == 'int64' and factor == math.floor(factor) then + fb.set_metric_value(exemplar, fb.int64(scaled)) + else + fb.set_metric_value(exemplar, scaled) + end + end + end + end + end + if unit then metric.unit = unit end +end + +function fb.extract_metric(metric, field, name, monotonic) + assert(field == 'count' or field == 'sum' or field == 'avg', 'unsupported extraction') + assert(metric.type == 'histogram' or metric.type == 'exponential_histogram' or metric.type == 'summary', + 'extraction requires a distribution') + local points = fb.array() + for i=1,metric.datapoints:len() do + local source = metric.datapoints:get(i) + local value = source:get(field == 'avg' and 'sum' or field) + local count = source:get('count') + if value and (field ~= 'avg' or count:number() ~= 0) then + local point = fb.map() + point:set('attributes', source:get('attributes')) + for _, key in ipairs({'time_unix_nano','start_time_unix_nano','flags'}) do + local v = source:get(key); if v then point:set(key, v) end + end + if field == 'avg' then fb.set_metric_value(point, value:number() / count:number()) + elseif field == 'count' then fb.set_metric_value(point, value) + else fb.set_metric_value(point, value:number()) end + points:append(point) + end + end + if points:len() == 0 then return nil end + -- The derived family has a fresh number-datapoint model, preserving source context. + local copy = metric:emit() + copy.name = name or metric.name .. '.' .. field + -- Distribution conversion is internal to this helper, never an implicit reinterpretation. + fb._fts_number_metric(copy, points, field, metric.aggregation_temporality or 'cumulative') + if field == 'count' then copy.unit = '1' end + if monotonic ~= nil and field ~= 'avg' then copy.is_monotonic = monotonic end + return copy +end + +function fb._fts_number_metric(metric, points, field, temporality) + local item = roots[metric]:get('item') + for _, key in ipairs(kinds) do item:remove(key) end + local data = fb.map(); data:set('data_points', points) + if field == 'avg' then item:set('gauge', data) + else + data:set('aggregation_temporality', fb.int64(({unspecified=0,delta=1,cumulative=2})[temporality or 'cumulative'])) + data:set('is_monotonic', field == 'count') + item:set('sum', data) + end +end + +local function unsigned_time(value) + assert(value and (value:type() == 'uint64' or value:type() == 'int64'), 'time must be an integer') + local text = value:value() + assert(text:sub(1,1) ~= '-', 'time must be nonnegative') + return text +end + +local function after_or_equal(a, b) + return #a > #b or (#a == #b and a >= b) +end + +local function bucket_total(values) + local total = fb.uint64(0) + assert(values:type() == 'array', 'bucket counts must be an array') + for i=1,values:len() do + total = fb._fts_add(total, fb.uint64(unsigned_time(values:get(i)))) + end + return total +end + +local function valid_id(value, size, optional) + assert(value and value:type() == 'bytes', 'ID must be bytes') + local text = value:value() + assert((optional and #text == 0) or (#text == size and not text:match('^%z+$')), 'invalid ID') +end + +local function typed_identity(value) + local kind, parts = value:type(), {} + if kind == 'map' then + local keys = value:keys(); table.sort(keys) + for _, key in ipairs(keys) do + local child = typed_identity(value:get(key)) + parts[#parts+1] = #key .. ':' .. key .. #child .. ':' .. child + end + elseif kind == 'array' then + for i=1,value:len() do + local child = typed_identity(value:get(i)) + parts[#parts+1] = #child .. ':' .. child + end + elseif kind == 'null' then + return 'null' + else + local text = kind == 'double' and string.format('%.17g', value:number()) or tostring(value:value()) + if kind == 'int64' or kind == 'uint64' then kind = 'integer' end + return kind .. ':' .. #text .. ':' .. text + end + return kind .. ':' .. table.concat(parts) +end + +function fb._fts_validate_telemetry(root, signal, original) + local item = root:get('item') + assert(root:get('resource'):get('attributes'):type() == 'map', 'resource attributes must be a map') + assert(root:get('scope'):get('attributes'):type() == 'map', 'scope attributes must be a map') + local before = original and original:get("item") + local function changed(key) + local a, b = item:get(key), before and before:get(key) + return not before or (a == nil) ~= (b == nil) or (a and not fb._fts_equal(a, b)) + end + if signal == 'traces' then + if changed("kind") then + local kind = item:get_number("kind") + assert(kind >= 0 and kind <= 5 and kind % 1 == 0, "invalid span kind") + end + if changed("status") then + local status = item:get("status") + local code = status and status:get_number("code") or 0 + assert(code >= 0 and code <= 2 and code % 1 == 0, "invalid span status") + end + for _, key in ipairs({'trace_id','span_id','parent_span_id'}) do + if changed(key) then + local value = item:get(key) + assert(value and value:type() == 'bytes', 'span ID must be bytes internally') + local text = value:value() + assert((key == 'parent_span_id' and #text == 0) or + (#text == (key == 'trace_id' and 16 or 8) and not text:match('^%z+$')), 'invalid span ID') + end + end + if changed("end_time_unix_nano") or changed("start_time_unix_nano") then + assert(after_or_equal(unsigned_time(item:get('end_time_unix_nano')), + unsigned_time(item:get('start_time_unix_nano'))), 'span ends before it starts') + end + assert(item:get('attributes'):type() == 'map', 'span attributes must be a map') + local events, links = item:get('events'), item:get('links') + assert(events and events:type() == 'array' and links and links:type() == 'array', + 'span events and links must be arrays') + for i=1,events:len() do + local event = events:get(i) + assert(event:get('attributes'):type() == 'map', 'event attributes must be a map') + unsigned_time(event:get('time_unix_nano')) + end + for i=1,links:len() do + local link = links:get(i) + valid_id(link:get('trace_id'), 16, false); valid_id(link:get('span_id'), 8, false) + assert(link:get('attributes'):type() == 'map', 'link attributes must be a map') + end + else + local kind, data = metric_data(item) + if changed('name') then + assert(item:get_string('name') ~= '', 'metric name must not be empty') + end + local points = data:get('data_points') + assert(points and points:type() == 'array', 'datapoints must be an array') + local previous_kind, previous_data + if before then previous_kind, previous_data = metric_data(before) end + local previous_points = previous_data and previous_data:get('data_points') + local layout, quantile_layout, identities = nil, nil, {} + for i=1,points:len() do + local point = points:get(i) + local previous_point = previous_kind == kind and previous_points and previous_points:get(i) + local point_changes = {} + local function point_changed(key) + if point_changes[key] ~= nil then return point_changes[key] end + local a, b = point:get(key), previous_point and previous_point:get(key) + local result = not previous_point or (a == nil) ~= (b == nil) or + (a ~= nil and not fb._fts_equal(a, b)) + point_changes[key] = result + return result + end + local attrs = point:get('attributes') + assert(attrs and attrs:type() == 'map', 'datapoint attributes must be a map') + for _, key in ipairs(attrs:keys()) do + assert(key ~= '' or (previous_point and + previous_point:get('attributes'):get(key)), 'metric attribute keys must not be empty') + end + local finish = unsigned_time(point:get('time_unix_nano')) + local start = unsigned_time(point:get('start_time_unix_nano')) + if point_changed('time_unix_nano') or point_changed('start_time_unix_nano') then + assert(after_or_equal(finish, start), 'datapoint ends before it starts') + end + local keys = attrs:keys(); table.sort(keys) + local identity = start .. ":" .. finish + for _, key in ipairs(keys) do + local value = typed_identity(attrs:get(key)) + identity = identity .. '|' .. #key .. ':' .. key .. #value .. ':' .. value + end + if point_changed('attributes') or point_changed('time_unix_nano') then + assert(not identities[identity], 'duplicate series/window; aggregate explicitly') + end + identities[identity] = true + if (kind == 'gauge' or kind == 'sum') and + (point_changed('as_int') or point_changed('as_double') or + (kind == 'sum' and (not previous_data or not + fb._fts_equal(data:get('is_monotonic'), previous_data:get('is_monotonic'))))) then + local integer, number = point:get('as_int'), point:get('as_double') + assert((integer ~= nil) ~= (number ~= nil), 'scalar datapoint needs exactly one value') + if kind == 'sum' and data:get('is_monotonic'):value() then + local value = integer or number + if value:type() == 'int64' then + assert(not value:value():match('^-'), 'monotonic sum cannot be negative') + elseif value:type() == 'double' then + assert(value:number() >= 0, 'monotonic sum cannot be negative') + end + end + end + if kind == 'histogram' then + local bounds, counts = point:get('explicit_bounds'), point:get('bucket_counts') + assert(bounds and bounds:type() == 'array' and counts and counts:type() == 'array', + 'histogram bounds and counts must be arrays') + if point_changed('explicit_bounds') or point_changed('bucket_counts') or point_changed('count') then + assert(counts:len() == bounds:len()+1, 'histogram bucket layout mismatch') + assert(bucket_total(counts):value() == unsigned_time(point:get('count')), 'histogram count mismatch') + end + local previous, current = -math.huge, {} + for j=1,bounds:len() do + local boundary = bounds:get(j):number() + if point_changed('explicit_bounds') then + assert(boundary > previous and boundary < math.huge, 'histogram bounds must increase') + end + previous = boundary; current[j] = boundary + end + if layout then + assert(#layout == #current, 'native histogram layouts must match') + for j=1,#layout do assert(layout[j] == current[j], 'native histogram layouts must match') end + else layout = current end + elseif kind == 'exponential_histogram' then + local positive, negative = point:get('positive'), point:get('negative') + local positive_counts = positive and positive:get('bucket_counts') or fb.array() + local negative_counts = negative and negative:get('bucket_counts') or fb.array() + assert(positive_counts:len() <= 65535 and negative_counts:len() <= 65535, 'too many exponential buckets') + local total = fb._fts_add(bucket_total(positive_counts), bucket_total(negative_counts)) + total = fb._fts_add(total, fb.uint64(unsigned_time(point:get('zero_count')))) + if point_changed('positive') or point_changed('negative') or + point_changed('zero_count') or point_changed('count') then + assert(total:value() == unsigned_time(point:get('count')), 'exponential histogram count mismatch') + end + local scale = point:get('scale'):number() + if point_changed('scale') then + assert(scale >= -10 and scale <= 20, 'unsupported exponential scale') + end + if point_changed('zero_threshold') then + assert(point:get_number('zero_threshold') >= 0, 'negative zero threshold') + end + elseif kind == 'summary' then + local quantiles = point:get('quantile_values') + assert(quantiles and quantiles:type() == 'array', 'summary quantiles must be an array') + local previous, current = -1, {} + for j=1,quantiles:len() do + local quantile = quantiles:get(j):get_number('quantile') + if point_changed('quantile_values') then + assert(quantile >= 0 and quantile <= 1 and quantile > previous, 'invalid summary quantile') + end + previous = quantile; current[j] = quantile + end + if quantile_layout then + assert(#quantile_layout == #current, 'native summary layouts must match') + for j=1,#current do + assert(quantile_layout[j] == current[j], 'native summary layouts must match') + end + else quantile_layout = current end + end + end + end +end + +local function clone(value) + local holder = fb.map(); holder:set('value', value); return holder:get('value') +end + +local function identity(point, keys) + local attrs = point:get('attributes') + if keys then fb.keep_keys(attrs, keys) end + local names = attrs:keys(); table.sort(names) + local parts = {unsigned_time(point:get('start_time_unix_nano')), unsigned_time(point:get('time_unix_nano'))} + for _, key in ipairs(names) do + local value = typed_identity(attrs:get(key)) + parts[#parts+1] = #key .. ':' .. key .. #value .. ':' .. value + end + return table.concat(parts, '|') +end + +function fb.aggregate_metric(metric, keys, operation) + operation = operation or 'sum' + assert(metric.type == 'gauge' or metric.type == 'sum' or metric.type == 'histogram', + 'aggregation supports gauges, sums and explicit histograms') + assert(operation == 'sum' or (metric.type == 'gauge' and + (operation == 'min' or operation == 'max' or operation == 'mean' or operation == 'count')), + 'unsupported aggregation for this metric type') + local groups, ordered = {}, {} + for i=1,metric.datapoints:len() do + local point = clone(metric.datapoints:get(i)) + local key = identity(point, keys) + local state = groups[key] + if not state then + state = {point=point, count=1}; groups[key] = state; ordered[#ordered+1] = state + else + local target = state.point + assert((point:get('flags') and point:get('flags'):value()) == + (target:get('flags') and target:get('flags'):value()), 'aggregation flags differ') + state.count = state.count + 1 + if metric.type == 'histogram' then + local bounds, old = point:get('explicit_bounds'), target:get('explicit_bounds') + assert(bounds:len() == old:len(), 'histogram layouts differ') + for j=1,bounds:len() do + assert(bounds:get(j):value() == old:get(j):value(), 'histogram layouts differ') + end + local buckets, previous = point:get('bucket_counts'), target:get('bucket_counts') + for j=1,buckets:len() do previous:set(j, fb._fts_add(previous:get(j), buckets:get(j))) end + target:set('count', fb._fts_add(target:get('count'), point:get('count'))) + if target:get('sum') and point:get('sum') then + target:set('sum', fb._fts_add(target:get('sum'), point:get('sum'))) + else target:remove('sum') end + for _, bound in ipairs({'min','max'}) do + local a, b = target:get_number(bound), point:get_number(bound) + if a and b then target:set(bound, (bound == 'min' and math.min or math.max)(a,b)) + else target:remove(bound) end + end + else + local a, b = fb.metric_value(target), fb.metric_value(point) + if operation == 'sum' or operation == 'mean' then + fb.set_metric_value(target, fb._fts_add(a,b)) + elseif operation == 'min' or operation == 'max' then + if (operation == 'min' and b:number() < a:number()) or + (operation == 'max' and b:number() > a:number()) then fb.set_metric_value(target,b) end + end + end + local exemplars = point:get('exemplars') + if exemplars then + local target_exemplars = target:get('exemplars') or fb.array() + for j=1,exemplars:len() do target_exemplars:append(exemplars:get(j)) end + target:set('exemplars', target_exemplars) + end + end + end + local output = fb.array() + for _, state in ipairs(ordered) do + if operation == 'mean' then fb.set_metric_value(state.point, fb.metric_value(state.point):number() / state.count) + elseif operation == 'count' then fb.set_metric_value(state.point, fb.int64(state.count)) end + output:append(state.point) + end + metric.datapoints = output + if operation == 'count' then metric.unit = '1' end +end + +function fb.merge_histogram_buckets(metric, bounds) + assert(metric.type == 'histogram', 'bucket merging requires an explicit histogram') + if type(bounds) == 'userdata' then + assert(bounds:type() == 'array', 'bounds must be an array') + local values = {} + for i=1,bounds:len() do values[i] = bounds:get(i):number() end + bounds = values + end + assert(type(bounds) == 'table', 'bounds must be a numeric array') + for i=2,#bounds do assert(bounds[i] > bounds[i-1], 'bounds must increase') end + for i=1,metric.datapoints:len() do + local point = metric.datapoints:get(i) + local old, counts = point:get('explicit_bounds'), point:get('bucket_counts') + local output, new_bounds = fb.array(), fb.array() + local at, total = 1, fb.uint64(0) + for _, boundary in ipairs(bounds) do + local found = false + while at <= old:len() and old:get(at):number() <= boundary do + total = fb._fts_add(total, counts:get(at)) + if old:get(at):number() == boundary then found = true end + at = at + 1 + end + assert(found, 'new bounds must be a subset of existing bounds') + new_bounds:append(boundary); output:append(total); total = fb.uint64(0) + end + while at <= counts:len() do total = fb._fts_add(total, counts:get(at)); at = at + 1 end + output:append(total) + point:set('explicit_bounds', new_bounds); point:set('bucket_counts', output) + end +end diff --git a/plugins/processor_fts/fts_traces.c b/plugins/processor_fts/fts_traces.c new file mode 100644 index 00000000000..8e660c34a51 --- /dev/null +++ b/plugins/processor_fts/fts_traces.c @@ -0,0 +1,143 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include "fts_telemetry.h" + +const struct fts_signal_adapter fts_traces_adapter = { + .event_type = FLB_PROCESSOR_TRACES, + .register_helpers = fts_register_telemetry_helpers, + .run_callback = fts_run_telemetry +}; + +/* + * Trace processors commit in place: the framework retains the original + * context pointer, so staged resource spans are moved into it. + */ +static void commit_trace(struct ctrace *input, struct ctrace *staged) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct ctrace_resource_span *resource; + struct ctrace_span *span; + + cfl_list_foreach_safe(head, tmp, &input->resource_spans) { + resource = cfl_list_entry(head, struct ctrace_resource_span, _head); + ctr_resource_span_destroy(resource); + } + + cfl_list_foreach_safe(head, tmp, &staged->resource_spans) { + resource = cfl_list_entry(head, struct ctrace_resource_span, _head); + cfl_list_del(head); + cfl_list_add(head, &input->resource_spans); + resource->ctx = input; + } + + cfl_list_foreach_safe(head, tmp, &staged->span_list) { + span = cfl_list_entry(head, struct ctrace_span, _head_global); + cfl_list_del(head); + cfl_list_add(head, &input->span_list); + span->ctx = input; + } + + input->last_span_id = staged->last_span_id; + ctr_destroy(staged); +} + +int fts_process_traces(struct flb_processor_instance *ins, struct ctrace *input, + struct ctrace **output, const char *tag, int tag_len) +{ + struct fts_runtime *ctx = ins->context; + cfl_sds_t encoded; + char *buffer = NULL; + size_t size; + size_t offset = 0; + int ret; + int handled_errors = 0; + const char *phase = "encode native traces"; + + *output = NULL; + + encoded = ctr_encode_opentelemetry_create(input); + if (!encoded) { + goto failure; + } + + phase = "transform span policies"; + ret = fts_telemetry_transform(ins, encoded, cfl_sds_len(encoded), &buffer, &size, + tag, tag_len); + ctr_encode_opentelemetry_destroy(encoded); + handled_errors = ret > 0; + if (ret < 0) { + goto failure; + } + + if (size == 0) { + flb_free(buffer); + *output = ctr_create(NULL); + if (!*output) { + return FLB_PROCESSOR_FAILURE; + } + commit_trace(input, *output); + *output = input; + + if (handled_errors) { + fts_report_policy_error(ins); + } + return FLB_PROCESSOR_SUCCESS; + } + + phase = "decode transformed traces"; + ret = ctr_decode_opentelemetry_create(output, buffer, size, &offset); + flb_free(buffer); + buffer = NULL; + if (ret != 0 || offset != size || !*output) { + ctr_destroy(*output); + *output = NULL; + goto failure; + } + + commit_trace(input, *output); + *output = input; + + if (handled_errors) { + fts_report_policy_error(ins); + } + + return FLB_PROCESSOR_SUCCESS; + +failure: + fts_report_policy_error(ins); + flb_free(buffer); + flb_plg_error(ins, "cannot construct FTS traces output (%s); edits discarded", phase); + + if (strcmp(ctx->error_mode, "drop") == 0) { + *output = ctr_create(NULL); + if (!*output) { + return FLB_PROCESSOR_FAILURE; + } + commit_trace(input, *output); + } + + *output = input; + return FLB_PROCESSOR_SUCCESS; +} diff --git a/plugins/processor_fts/fts_value_helpers.lua b/plugins/processor_fts/fts_value_helpers.lua new file mode 100644 index 00000000000..50c6bf10288 --- /dev/null +++ b/plugins/processor_fts/fts_value_helpers.lua @@ -0,0 +1,37 @@ +local fb = package.loaded.fluentbit +function fb.keep_keys(map, keys) + local keep = {}; for _, k in ipairs(keys) do keep[k] = true end + for _, k in ipairs(map:keys()) do if not keep[k] then map:remove(k) end end +end +function fb.parse_time(text) + assert(type(text) == 'string', 'timestamp must be RFC3339 text') + local y,m,d,h,mi,s,tail = text:match('^(%d%d%d%d)%-(%d%d)%-(%d%d)[Tt](%d%d):(%d%d):(%d%d)(.*)$') + assert(y, 'invalid RFC3339 timestamp') + y,m,d,h,mi,s = tonumber(y),tonumber(m),tonumber(d),tonumber(h),tonumber(mi),tonumber(s) + local leap = y%4 == 0 and (y%100 ~= 0 or y%400 == 0) + local lengths = {31,leap and 29 or 28,31,30,31,30,31,31,30,31,30,31} + assert(y>=1969 and y<=2107 and m>=1 and m<=12 and d>=1 and d<=lengths[m] + and h<24 and mi<60 and s<60, 'invalid calendar timestamp') + local fraction = tail:match('^%.(%d+)') + local nsec = 0 + if fraction then + assert(#fraction <= 9, 'timestamp exceeds nanosecond precision') + nsec = tonumber(fraction .. string.rep('0', 9 - #fraction)) + tail = tail:sub(#fraction + 2) + end + local offset = 0 + if tail ~= 'Z' and tail ~= 'z' then + local sign,oh,om = tail:match('^([+-])(%d%d):(%d%d)$') + assert(sign and tonumber(oh)<24 and tonumber(om)<60, 'invalid timestamp timezone') + offset = (tonumber(oh)*60 + tonumber(om))*60*(sign == '+' and 1 or -1) + end + local days = 0 + if y == 1969 then days = -365 end + for year=1970,y-1 do + days = days + ((year%4 == 0 and (year%100 ~= 0 or year%400 == 0)) and 366 or 365) + end + for month=1,m-1 do days = days + lengths[month] end + local sec = (days+d-1)*86400+h*3600+mi*60+s-offset + assert(sec>=0 and sec<=4294967293, 'timestamp outside log event range') + return {sec=sec,nsec=nsec} +end diff --git a/plugins/processor_fts/fts_values.c b/plugins/processor_fts/fts_values.c new file mode 100644 index 00000000000..9a112908087 --- /dev/null +++ b/plugins/processor_fts/fts_values.c @@ -0,0 +1,2173 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fts.h" +#include "fts_helpers.h" + +#define VALUE_MT "fluentbit.typed_value" +#define SAFE_INTEGER 9007199254740991.0 + +/* Empty input maps need no native allocation until a policy writes to them. + * This immutable backing is never attached to a mutable CFL container. */ +static struct cfl_kvlist empty_map = { + .list = {&empty_map.list, &empty_map.list} +}; +static struct cfl_variant empty_value = { + .type = CFL_VARIANT_KVLIST, + .data.as_kvlist = &empty_map +}; + +static struct fts_value *push_value(lua_State *l, struct cfl_variant *v, int readonly, + int borrowed); + +/* Record handles may borrow chunk storage, so they are valid only during the + * callback that created them. copy() returns a value that can be kept. */ +static struct fts_value *check_live(lua_State *l, struct fts_value *value) +{ + if (value->record && value->generation != value->ctx->generation) { + luaL_error(l, "log value used after its callback; use copy() to keep it"); + } + + return value; +} + +/* Only push_value() creates userdata of this exact size, and it stamps the + * private runtime pointer; policies cannot forge either. Comparing them avoids + * a metatable round trip on every typed call. */ +static struct fts_value *check_value(lua_State *l, int index) +{ + struct fts_runtime *ctx; + struct fts_value *value; + + lua_getallocf(l, (void **) &ctx); + + value = NULL; + if (lua_type(l, index) == LUA_TUSERDATA && lua_objlen(l, index) == sizeof(*value)) { + value = lua_touserdata(l, index); + } + + if (!value || value->ctx != ctx) { + luaL_argerror(l, index, "typed value expected"); + return NULL; + } + + check_live(l, value); + + return value; +} + +/* Only adapter callbacks call these: their closures carry the key strings + * as upvalues, in fts_key order. */ +void fts_getfield(lua_State *l, int index, enum fts_key key) +{ + if (index < 0 && index > LUA_REGISTRYINDEX) { + index = lua_gettop(l) + index + 1; + } + + lua_pushvalue(l, lua_upvalueindex(key + 1)); + lua_rawget(l, index); +} + +void fts_setfield(lua_State *l, int index, enum fts_key key) +{ + if (index < 0 && index > LUA_REGISTRYINDEX) { + index = lua_gettop(l) + index + 1; + } + + lua_pushvalue(l, lua_upvalueindex(key + 1)); + lua_insert(l, -2); + lua_rawset(l, index); +} + +/* Compare stored entries, preserving duplicate keys and their order. */ +int fts_values_equal(struct cfl_variant *a, struct cfl_variant *b, int depth) +{ + struct cfl_list *left; + struct cfl_list *right; + struct cfl_kvpair *x; + struct cfl_kvpair *y; + size_t i; + + if (a == b) { + return FLB_TRUE; + } + + if (!a || !b || a->type != b->type || depth >= FTS_MAX_DEPTH) { + return FLB_FALSE; + } + + switch (a->type) { + case CFL_VARIANT_NULL: + return FLB_TRUE; + case CFL_VARIANT_BOOL: + return a->data.as_bool == b->data.as_bool; + case CFL_VARIANT_INT: + return a->data.as_int64 == b->data.as_int64; + case CFL_VARIANT_UINT: + return a->data.as_uint64 == b->data.as_uint64; + case CFL_VARIANT_DOUBLE: + return memcmp(&a->data.as_double, &b->data.as_double, sizeof(double)) == 0; + case CFL_VARIANT_STRING: + case CFL_VARIANT_BYTES: + return cfl_variant_size_get(a) == cfl_variant_size_get(b) && + memcmp(a->data.as_string, b->data.as_string, cfl_variant_size_get(a)) == 0; + case CFL_VARIANT_ARRAY: + if (a->data.as_array->entry_count != b->data.as_array->entry_count) { + return FLB_FALSE; + } + + for (i = 0; i < a->data.as_array->entry_count; i++) { + if (!fts_values_equal(a->data.as_array->entries[i], + b->data.as_array->entries[i], depth + 1)) { + return FLB_FALSE; + } + } + + return FLB_TRUE; + case CFL_VARIANT_KVLIST: + left = a->data.as_kvlist->list.next; + right = b->data.as_kvlist->list.next; + + while (left != &a->data.as_kvlist->list && right != &b->data.as_kvlist->list) { + x = cfl_list_entry(left, struct cfl_kvpair, _head); + y = cfl_list_entry(right, struct cfl_kvpair, _head); + if (cfl_sds_len(x->key) != cfl_sds_len(y->key) || + memcmp(x->key, y->key, cfl_sds_len(x->key)) != 0 || + !fts_values_equal(x->val, y->val, depth + 1)) { + return FLB_FALSE; + } + + left = left->next; + right = right->next; + } + + return left == &a->data.as_kvlist->list && right == &b->data.as_kvlist->list; + default: + return FLB_FALSE; + } +} + +static int equal_values(lua_State *l) +{ + lua_pushboolean(l, fts_values_equal(fts_resolve_value(l, 1, 0), + fts_resolve_value(l, 2, 0), 0)); + return 1; +} + +/* Arena copies are private native results. Script-visible snapshots use heap + * ownership so retained Lua values remain independent of chunk lifetimes. */ +struct cfl_variant *fts_copy_value_in(struct cfl_variant *v, int depth, struct cfl_arena *arena) +{ + struct cfl_variant *copy; + struct cfl_variant *child; + struct cfl_array *array; + struct cfl_kvlist *map; + struct cfl_list *head; + struct cfl_kvpair *pair; + size_t i; + size_t length; + + if (!v || depth >= FTS_MAX_DEPTH) { + return NULL; + } + + switch (v->type) { + case CFL_VARIANT_NULL: + return cfl_variant_create_from_null_in(arena); + case CFL_VARIANT_BOOL: + return cfl_variant_create_from_bool_in(arena, v->data.as_bool); + case CFL_VARIANT_INT: + return cfl_variant_create_from_int64_in(arena, v->data.as_int64); + case CFL_VARIANT_UINT: + return cfl_variant_create_from_uint64_in(arena, v->data.as_uint64); + case CFL_VARIANT_DOUBLE: + return cfl_variant_create_from_double_in(arena, v->data.as_double); + case CFL_VARIANT_STRING: + case CFL_VARIANT_BYTES: + if (arena) { + if (v->type == CFL_VARIANT_STRING) { + return cfl_variant_create_from_string_s_in(arena, v->data.as_string, + cfl_variant_size_get(v), CFL_FALSE); + } + return cfl_variant_create_from_bytes_in(arena, v->data.as_bytes, + cfl_variant_size_get(v), CFL_FALSE); + } + + /* Consumers outside FTS read string and bytes lengths through cfl_sds_len(), + * so heap copies always own a real sds payload. */ + length = cfl_variant_size_get(v); + if (length > INT_MAX) { + return NULL; + } + + if (v->type == CFL_VARIANT_STRING) { + copy = cfl_variant_create_from_string_s(v->data.as_string, length, CFL_FALSE); + } + else { + copy = cfl_variant_create_from_bytes(v->data.as_bytes, length, CFL_FALSE); + } + + if (!copy) { + flb_errno(); + } + return copy; + case CFL_VARIANT_ARRAY: + array = cfl_array_create_in(arena, v->data.as_array->entry_count); + if (!array) { + return NULL; + } + + for (i = 0; i < v->data.as_array->entry_count; i++) { + child = fts_copy_value_in(v->data.as_array->entries[i], depth + 1, arena); + if (!child || cfl_array_append(array, child) != 0) { + cfl_variant_destroy(child); + cfl_array_destroy(array); + return NULL; + } + } + + copy = cfl_variant_create_from_array_in(arena, array); + if (!copy) { + cfl_array_destroy(array); + } + return copy; + case CFL_VARIANT_KVLIST: + map = cfl_kvlist_create_in(arena); + if (!map) { + return NULL; + } + + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + child = fts_copy_value_in(pair->val, depth + 1, arena); + if (!child || + cfl_kvlist_insert_s(map, pair->key, cfl_sds_len(pair->key), child) != 0) { + cfl_variant_destroy(child); + cfl_kvlist_destroy(map); + return NULL; + } + } + + copy = cfl_variant_create_from_kvlist_in(arena, map); + if (!copy) { + cfl_kvlist_destroy(map); + } + return copy; + default: + return NULL; + } +} + +struct cfl_variant *fts_copy_value(struct cfl_variant *value, int depth) +{ + return fts_copy_value_in(value, depth, NULL); +} + +/* Conservative accounting includes container overhead and every owned scalar. */ +size_t fts_value_cost(struct cfl_variant *v, int depth) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + size_t total = 256; + size_t child; + size_t i; + + if (!v || depth >= FTS_MAX_DEPTH) { + return SIZE_MAX; + } + + if (v->type == CFL_VARIANT_STRING || v->type == CFL_VARIANT_BYTES) { + child = cfl_variant_size_get(v); + return child > SIZE_MAX - total ? SIZE_MAX : total + child; + } + + if (v->type == CFL_VARIANT_ARRAY) { + for (i = 0; i < v->data.as_array->entry_count; i++) { + child = fts_value_cost(v->data.as_array->entries[i], depth + 1); + if (child > SIZE_MAX - total) { + return SIZE_MAX; + } + total += child; + } + } + else if (v->type == CFL_VARIANT_KVLIST) { + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + child = cfl_sds_len(pair->key) + 128; + if (child > SIZE_MAX - total) { + return SIZE_MAX; + } + total += child; + + child = fts_value_cost(pair->val, depth + 1); + if (child > SIZE_MAX - total) { + return SIZE_MAX; + } + total += child; + } + } + + return total; +} + +void fts_charge_value(lua_State *l, struct fts_value *owner, size_t bytes) +{ + struct fts_runtime *ctx = owner->ctx; + + if (bytes > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - bytes || + ctx->lua_bytes > ctx->memory_limit - ctx->native_bytes - bytes) { + /* Native values are invisible to Lua's allocation-based GC pacing. */ + lua_gc(l, LUA_GCCOLLECT, 0); + } + + if (bytes > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - bytes || + ctx->lua_bytes > ctx->memory_limit - ctx->native_bytes - bytes) { + luaL_error(l, "Lua policy memory limit exceeded"); + } + + /* In-place additions become chunk data when the record callback ends. */ + if (owner->inplace) { + ctx->undo_charged += bytes; + } + else { + owner->charged += bytes; + } + + ctx->native_bytes += bytes; + if (bytes > SIZE_MAX - ctx->native_allocated) { + ctx->native_allocated = SIZE_MAX; + } + else { + ctx->native_allocated = ctx->native_allocated + bytes; + } +} + +/* No native ownership or reservation means no finalizer is needed. + * A first write installs normal ownership before allocating storage. */ +struct fts_value *fts_push_empty_map(lua_State *l) +{ + return fts_push_borrowed(l, &empty_value); +} + +/* Record roots read the chunk's tree directly; the first write copies it. */ +struct fts_value *fts_push_borrowed(lua_State *l, struct cfl_variant *source) +{ + struct fts_value *value; + + value = push_value(l, source, FLB_FALSE, FLB_TRUE); + value->record = FLB_TRUE; + value->generation = value->ctx->generation; + + return value; +} + +/* Values written into a container come from the same allocator as it. */ +static struct cfl_arena *container_arena(struct cfl_variant *container) +{ + if (container->type == CFL_VARIANT_KVLIST) { + return container->data.as_kvlist->arena; + } + + if (container->type == CFL_VARIANT_ARRAY) { + return container->data.as_array->arena; + } + + return NULL; +} + +/* Exact byte matching: OTel attribute names are case sensitive. */ +struct cfl_kvpair *fts_find_pair(struct cfl_variant *v, const char *key, size_t length) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + + if (v->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (cfl_sds_len(pair->key) == length && memcmp(pair->key, key, length) == 0) { + return pair; + } + } + + return NULL; +} + +/* Views retain their parent through their Lua environment. Resolve each use + * rather than storing child pointers that set/remove could invalidate. */ +struct cfl_variant *fts_resolve_value(lua_State *l, int index, int depth) +{ + struct fts_value *value; + struct cfl_variant *parent; + struct cfl_variant *result = NULL; + struct cfl_kvpair *pair; + const char *key; + size_t length; + lua_Number position; + + value = check_value(l, index); + if (value->owned) { + return value->owned; + } + + if (value->cache && value->epoch == value->root->epoch) { + check_live(l, value->root); + return value->cache; + } + + if (depth >= FTS_MAX_DEPTH || !lua_checkstack(l, 4)) { + luaL_error(l, "typed value nesting exceeds limit"); + } + + lua_getfenv(l, index); + lua_rawgeti(l, -1, 1); + parent = fts_resolve_value(l, -1, depth + 1); + lua_pop(l, 1); + + lua_rawgeti(l, -1, 2); + if (parent->type == CFL_VARIANT_KVLIST && lua_type(l, -1) == LUA_TSTRING) { + key = lua_tolstring(l, -1, &length); + pair = fts_find_pair(parent, key, length); + if (pair) { + result = pair->val; + } + } + else if (parent->type == CFL_VARIANT_ARRAY && lua_type(l, -1) == LUA_TNUMBER) { + position = lua_tonumber(l, -1); + if (position >= 1 && position <= parent->data.as_array->entry_count) { + result = parent->data.as_array->entries[(size_t) position - 1]; + } + } + lua_pop(l, 2); + + if (!result) { + luaL_error(l, "typed value path no longer exists"); + } + + value->cache = result; + value->epoch = value->root->epoch; + + return result; +} + +static struct fts_value *push_value(lua_State *l, struct cfl_variant *v, int readonly, + int borrowed) +{ + struct fts_value *value; + + value = lua_newuserdata(l, sizeof(*value)); + value->owned = v; + value->readonly = readonly; + value->dirty = FLB_FALSE; + value->depth_changed = !borrowed; + value->borrowed = borrowed; + value->record = FLB_FALSE; + value->inplace = FLB_FALSE; + value->generation = 0; + value->charged = 0; + value->root = NULL; + value->cache = NULL; + value->epoch = 0; + lua_getallocf(l, (void **) &value->ctx); + + /* Borrowed storage is not owned, so it needs no finalizer. */ + if (borrowed) { + lua_rawgeti(l, LUA_REGISTRYINDEX, value->ctx->empty_metatable_ref); + } + else { + lua_rawgeti(l, LUA_REGISTRYINDEX, value->ctx->value_metatable_ref); + } + lua_setmetatable(l, -2); + + return value; +} + +struct fts_value *fts_push_owned(lua_State *l, struct cfl_variant *v, int readonly) +{ + return push_value(l, v, readonly, FLB_FALSE); +} + +static void push_view(lua_State *l, int parent, int key) +{ + struct fts_value *owner; + struct fts_value *view; + + owner = check_value(l, parent); + if (!owner->owned && !owner->root) { + luaL_error(l, "typed value has no storage"); + } + + /* Views own no storage, so they use the finalizer-free metatable. */ + view = push_value(l, NULL, owner->readonly, FLB_TRUE); + view->borrowed = FLB_FALSE; + view->depth_changed = FLB_TRUE; + view->root = owner->owned ? owner : owner->root; + + /* The environment pins the parent; slots 1 and 2 hold parent and key. */ + lua_createtable(l, 2, 0); + lua_pushvalue(l, parent); + lua_rawseti(l, -2, 1); + lua_pushvalue(l, key); + lua_rawseti(l, -2, 2); + lua_setfenv(l, -2); +} + +static int value_gc(lua_State *l) +{ + struct fts_value *value; + + /* Only the VM can call this private finalizer. */ + value = lua_touserdata(l, 1); + if (!value->borrowed) { + cfl_variant_destroy(value->owned); + } + + value->owned = NULL; + value->ctx->native_bytes -= value->charged; + value->charged = 0; + + return 0; +} + +static int value_type(lua_State *l) +{ + struct cfl_variant *v; + const char *name; + + v = fts_resolve_value(l, 1, 0); + switch (v->type) { + case CFL_VARIANT_NULL: + name = "null"; + break; + case CFL_VARIANT_BOOL: + name = "bool"; + break; + case CFL_VARIANT_INT: + name = "int64"; + break; + case CFL_VARIANT_UINT: + name = "uint64"; + break; + case CFL_VARIANT_DOUBLE: + name = "double"; + break; + case CFL_VARIANT_STRING: + name = "string"; + break; + case CFL_VARIANT_BYTES: + name = "bytes"; + break; + case CFL_VARIANT_ARRAY: + name = "array"; + break; + case CFL_VARIANT_KVLIST: + name = "map"; + break; + default: + return luaL_error(l, "unsupported telemetry type"); + } + + lua_pushstring(l, name); + return 1; +} + +static int push_scalar(lua_State *l, struct cfl_variant *v) +{ + char buffer[32]; + + switch (v->type) { + case CFL_VARIANT_NULL: + lua_pushnil(l); + break; + case CFL_VARIANT_BOOL: + lua_pushboolean(l, v->data.as_bool); + break; + case CFL_VARIANT_DOUBLE: + lua_pushnumber(l, v->data.as_double); + break; + case CFL_VARIANT_STRING: + lua_pushlstring(l, v->data.as_string, cfl_variant_size_get(v)); + break; + case CFL_VARIANT_BYTES: + lua_pushlstring(l, v->data.as_bytes, cfl_variant_size_get(v)); + break; + case CFL_VARIANT_INT: + snprintf(buffer, sizeof(buffer), "%" PRId64, v->data.as_int64); + lua_pushstring(l, buffer); + break; + case CFL_VARIANT_UINT: + snprintf(buffer, sizeof(buffer), "%" PRIu64, v->data.as_uint64); + lua_pushstring(l, buffer); + break; + default: + return luaL_error(l, "value() requires a scalar; use get() for containers"); + } + + return 1; +} + +static int push_number(lua_State *l, struct cfl_variant *v) +{ + lua_Number number; + + if (v->type == CFL_VARIANT_INT) { + if (v->data.as_int64 < -INT64_C(9007199254740991) || + v->data.as_int64 > INT64_C(9007199254740991)) { + return luaL_error(l, "integer cannot be represented exactly as a Lua number"); + } + number = v->data.as_int64; + } + else if (v->type == CFL_VARIANT_UINT) { + if (v->data.as_uint64 > UINT64_C(9007199254740991)) { + return luaL_error(l, "integer cannot be represented exactly as a Lua number"); + } + number = v->data.as_uint64; + } + else if (v->type == CFL_VARIANT_DOUBLE) { + number = v->data.as_double; + } + else { + return luaL_error(l, "number() requires a numeric value"); + } + + lua_pushnumber(l, number); + return 1; +} + +static int value_scalar(lua_State *l) +{ + return push_scalar(l, fts_resolve_value(l, 1, 0)); +} + +static int value_number(lua_State *l) +{ + return push_number(l, fts_resolve_value(l, 1, 0)); +} + +static size_t array_index(lua_State *l, struct cfl_variant *v, int index) +{ + lua_Number position; + + if (lua_type(l, index) != LUA_TNUMBER) { + luaL_error(l, "array index must be a number"); + } + + position = lua_tonumber(l, index); + if (!isfinite(position) || position != floor(position) || position < 1 || + position > v->data.as_array->entry_count) { + luaL_error(l, "array index out of bounds (indices start at 1)"); + } + + return (size_t) position - 1; +} + +/* Resolve once without creating a Lua view for scalar reads. */ +static struct cfl_variant *get_child(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_kvpair *pair; + const char *key; + size_t length; + + v = fts_resolve_value(l, 1, 0); + if (v->type == CFL_VARIANT_KVLIST) { + if (lua_type(l, 2) != LUA_TSTRING) { + luaL_error(l, "map key must be a string"); + } + + key = lua_tolstring(l, 2, &length); + pair = fts_find_pair(v, key, length); + return pair ? pair->val : NULL; + } + + if (v->type == CFL_VARIANT_ARRAY) { + return v->data.as_array->entries[array_index(l, v, 2)]; + } + + luaL_error(l, "get() requires a map or array"); + return NULL; +} + +static int value_get(lua_State *l) +{ + if (!get_child(l)) { + lua_pushnil(l); + return 1; + } + + push_view(l, 1, 2); + return 1; +} + +/* Scalar convenience accessors retain strict types and missing-key semantics. */ +static int get_scalar(lua_State *l, int type) +{ + struct cfl_variant *v; + + v = get_child(l); + if (!v) { + lua_pushnil(l); + return 1; + } + + if (type == CFL_VARIANT_DOUBLE) { + return push_number(l, v); + } + + if (v->type != type) { + return luaL_error(l, "field has a different type than the requested scalar accessor"); + } + + return push_scalar(l, v); +} + +static int value_get_string(lua_State *l) +{ + return get_scalar(l, CFL_VARIANT_STRING); +} + +static int value_get_bool(lua_State *l) +{ + return get_scalar(l, CFL_VARIANT_BOOL); +} + +static int value_get_number(lua_State *l) +{ + return get_scalar(l, CFL_VARIANT_DOUBLE); +} + +enum { + UNDO_SET_PAIR, + UNDO_INSERT_PAIR, + UNDO_REMOVE_PAIR, + UNDO_SET_INDEX, + UNDO_APPEND +}; + +/* Reserve before mutating so a recorded edit can never be lost. */ +static void undo_reserve(lua_State *l, struct fts_runtime *ctx) +{ + struct fts_undo_entry *entries; + size_t size; + + if (ctx->undo_count < ctx->undo_size) { + return; + } + + size = ctx->undo_size ? ctx->undo_size * 2 : 64; + if (size > SIZE_MAX / sizeof(*entries)) { + luaL_error(l, "cannot allocate edit log"); + } + + entries = flb_realloc(ctx->undo, size * sizeof(*entries)); + if (!entries) { + flb_errno(); + luaL_error(l, "cannot allocate edit log"); + } + + ctx->undo = entries; + ctx->undo_size = size; +} + +static void undo_add(struct fts_runtime *ctx, int type, struct cfl_list *list, + struct cfl_kvpair *pair, struct cfl_list *prev, + struct cfl_array *array, size_t index, struct cfl_variant *old) +{ + struct fts_undo_entry *entry; + struct cfl_variant *held_value = old; + size_t held = 64; + size_t cost; + + entry = &ctx->undo[ctx->undo_count++]; + entry->type = type; + entry->list = list; + entry->pair = pair; + entry->prev = prev; + entry->array = array; + entry->index = index; + entry->old = old; + + if (type == UNDO_REMOVE_PAIR) { + held += 128 + cfl_sds_len(pair->key); + held_value = pair->val; + } + + if (held_value) { + cost = fts_value_cost(held_value, 0); + held = cost > SIZE_MAX - held ? SIZE_MAX : held + cost; + } + + ctx->undo_held = held > SIZE_MAX - ctx->undo_held ? SIZE_MAX : ctx->undo_held + held; +} + +/* Restore edits newer than mark, newest first. */ +void fts_undo_rollback(struct fts_runtime *ctx, size_t mark) +{ + struct fts_undo_entry *entry; + struct cfl_variant *current; + + while (ctx->undo_count > mark) { + entry = &ctx->undo[--ctx->undo_count]; + switch (entry->type) { + case UNDO_SET_PAIR: + current = entry->pair->val; + entry->pair->val = entry->old; + cfl_variant_destroy(current); + break; + case UNDO_INSERT_PAIR: + cfl_kvpair_destroy(entry->pair); + break; + case UNDO_REMOVE_PAIR: + /* cfl_list_add_after() leaves next->prev stale; insert before next. */ + cfl_list_add_before(&entry->pair->_head, entry->prev->next, entry->list); + break; + case UNDO_SET_INDEX: + current = entry->array->entries[entry->index]; + entry->array->entries[entry->index] = entry->old; + cfl_variant_destroy(current); + break; + case UNDO_APPEND: + current = entry->array->entries[--entry->array->entry_count]; + entry->array->entries[entry->array->entry_count] = NULL; + cfl_variant_destroy(current); + break; + } + } + + if (mark == 0) { + ctx->undo_held = 0; + } +} + +/* Release replaced and removed values once the chunk output is final. */ +void fts_undo_commit(struct fts_runtime *ctx) +{ + struct fts_undo_entry *entry; + size_t index; + + for (index = 0; index < ctx->undo_count; index++) { + entry = &ctx->undo[index]; + if (entry->type == UNDO_SET_PAIR || entry->type == UNDO_SET_INDEX) { + cfl_variant_destroy(entry->old); + } + else if (entry->type == UNDO_REMOVE_PAIR) { + cfl_kvpair_destroy(entry->pair); + } + } + + ctx->undo_count = 0; + ctx->undo_held = 0; +} + +static struct fts_value *root_value(lua_State *l, int index) +{ + struct fts_value *value; + + /* Public mutation entry points already verified the receiver. View + * environments are private and retain only verified parent values. */ + value = lua_touserdata(l, index); + return value->owned ? value : value->root; +} + +static size_t cost_add(size_t a, size_t b) +{ + return b > SIZE_MAX - a ? SIZE_MAX : a + b; +} + +static void credit_value(lua_State *l, struct fts_value *owner, size_t bytes) +{ + /* Replaced in-place data stays allocated in the undo log until commit. */ + if (owner->inplace) { + return; + } + + /* An unknown cost never releases reservations held by other values. */ + if (bytes == SIZE_MAX) { + return; + } + + if (bytes > owner->charged) { + bytes = owner->charged; + } + + owner->charged -= bytes; + owner->ctx->native_bytes -= bytes; +} + +static void charge_mutation(lua_State *l, int index, size_t overhead) +{ + size_t bytes = 256; + size_t length; + + if (lua_type(l, index) == LUA_TUSERDATA) { + bytes = fts_value_cost(fts_resolve_value(l, index, 0), 0); + if (bytes == SIZE_MAX) { + luaL_error(l, "typed value nesting exceeds limit"); + } + } + else if (lua_type(l, index) == LUA_TSTRING) { + lua_tolstring(l, index, &length); + bytes += length; + } + else if (lua_type(l, index) == LUA_TNUMBER) { + if (!isfinite(lua_tonumber(l, index))) { + luaL_error(l, "telemetry numbers must be finite"); + } + } + else if (lua_type(l, index) != LUA_TBOOLEAN) { + luaL_error(l, "use typed telemetry values and fb.null"); + } + + fts_charge_value(l, root_value(l, 1), cost_add(bytes, overhead)); +} + +static struct cfl_variant *from_lua(lua_State *l, int index, struct cfl_arena *arena) +{ + const char *str; + size_t length; + lua_Number number; + struct cfl_variant *v; + + switch (lua_type(l, index)) { + case LUA_TUSERDATA: + v = fts_copy_value_in(fts_resolve_value(l, index, 0), 0, arena); + break; + case LUA_TBOOLEAN: + v = cfl_variant_create_from_bool_in(arena, lua_toboolean(l, index)); + break; + case LUA_TSTRING: + str = lua_tolstring(l, index, &length); + v = cfl_variant_create_from_string_s_in(arena, (char *) str, length, CFL_FALSE); + break; + case LUA_TNUMBER: + number = lua_tonumber(l, index); + if (!isfinite(number)) { + luaL_error(l, "number must be finite"); + } + v = cfl_variant_create_from_double_in(arena, number); + break; + default: + luaL_error(l, "use a scalar or typed value; use fb.null instead of nil"); + return NULL; + } + + if (!v) { + luaL_error(l, "cannot copy typed value (allocation or nesting limit)"); + } + + return v; +} + +static void mark_dirty(lua_State *l, int index) +{ + struct fts_value *value; + + value = check_value(l, index); + if (!value->owned) { + value = value->root; + check_live(l, value); + } + + value->dirty = FLB_TRUE; +} + +/* Copy a borrowed root before its first write; views address it by path. */ +static void detach_root(lua_State *l, int index, int depth) +{ + struct fts_value *value; + struct cfl_variant *copy; + size_t cost; + + value = lua_touserdata(l, index); + if (!value->owned) { + if (depth >= FTS_MAX_DEPTH || !lua_checkstack(l, 4)) { + luaL_error(l, "typed value nesting exceeds limit"); + } + + lua_getfenv(l, index); + lua_rawgeti(l, -1, 1); + detach_root(l, lua_gettop(l), depth + 1); + lua_pop(l, 2); + return; + } + + if (!value->borrowed || value->inplace) { + return; + } + + check_live(l, value); + + /* + * Chunk trees are edited in place with an undo log. Written values are + * allocated from the container's own arena, if any. Shared empty maps + * are immutable. + */ + if (value->ctx->inplace_ok && value->owned != &empty_value) { + value->inplace = FLB_TRUE; + return; + } + + cost = fts_value_cost(value->owned, 0); + if (cost == SIZE_MAX) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + + /* The finalizer skips borrowed storage, so charges are released on failure. */ + lua_rawgeti(l, LUA_REGISTRYINDEX, value->ctx->value_metatable_ref); + lua_setmetatable(l, index); + if (value->charged == 0) { + fts_charge_value(l, value, cost); + } + + copy = fts_copy_value(value->owned, 0); + if (!copy) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + + value->owned = copy; + value->borrowed = FLB_FALSE; +} + +static struct cfl_variant *writable_value(lua_State *l) +{ + struct fts_value *value; + + value = check_value(l, 1); + if (value->readonly) { + luaL_error(l, "typed value is read-only"); + } + + detach_root(l, 1, 0); + (value->owned ? value : value->root)->epoch++; + + if (value->owned) { + value->dirty = FLB_TRUE; + return value->owned; + } + + mark_dirty(l, 1); + return fts_resolve_value(l, 1, 0); +} + +static int value_set(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_variant *replacement; + struct cfl_variant *old; + struct cfl_kvpair *pair; + struct cfl_kvpair *duplicate; + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_list *prev; + struct fts_value *self; + struct fts_value *root; + const char *key; + size_t length; + size_t position; + + v = writable_value(l); + root = root_value(l, 1); + if (root->inplace) { + undo_reserve(l, root->ctx); + } + + self = lua_touserdata(l, 1); + if (!self->owned || lua_type(l, 3) == LUA_TUSERDATA) { + root_value(l, 1)->depth_changed = FLB_TRUE; + } + + if (v->type == CFL_VARIANT_KVLIST) { + if (lua_type(l, 2) != LUA_TSTRING) { + return luaL_error(l, "map key must be a string"); + } + + key = lua_tolstring(l, 2, &length); + pair = fts_find_pair(v, key, length); + charge_mutation(l, 3, pair ? 0 : 128 + length); + replacement = from_lua(l, 3, container_arena(v)); + + if (pair) { + if (cfl_container_move_variant_to_kvlist(v->data.as_kvlist, replacement) != 0) { + cfl_variant_destroy(replacement); + return luaL_error(l, "cannot attach map value"); + } + + old = pair->val; + pair->val = replacement; + if (root->inplace) { + undo_add(root->ctx, UNDO_SET_PAIR, NULL, pair, NULL, NULL, 0, old); + } + else { + credit_value(l, root, fts_value_cost(old, 0)); + cfl_variant_destroy(old); + } + + cfl_list_foreach_safe(head, tmp, &v->data.as_kvlist->list) { + duplicate = cfl_list_entry(head, struct cfl_kvpair, _head); + if (duplicate != pair && cfl_sds_len(duplicate->key) == length && + memcmp(duplicate->key, key, length) == 0) { + if (root->inplace) { + undo_reserve(l, root->ctx); + prev = duplicate->_head.prev; + cfl_list_del(&duplicate->_head); + undo_add(root->ctx, UNDO_REMOVE_PAIR, &v->data.as_kvlist->list, + duplicate, prev, NULL, 0, NULL); + } + else { + credit_value(l, root, cost_add(128 + length, + fts_value_cost(duplicate->val, 0))); + cfl_kvpair_destroy(duplicate); + } + } + } + } + else if (cfl_kvlist_insert_s(v->data.as_kvlist, (char *) key, length, replacement) != 0) { + cfl_variant_destroy(replacement); + return luaL_error(l, "cannot insert map value"); + } + else if (root->inplace) { + undo_add(root->ctx, UNDO_INSERT_PAIR, NULL, + cfl_list_entry_last(&v->data.as_kvlist->list, struct cfl_kvpair, _head), + NULL, NULL, 0, NULL); + } + } + else if (v->type == CFL_VARIANT_ARRAY) { + position = array_index(l, v, 2); + charge_mutation(l, 3, 0); + replacement = from_lua(l, 3, container_arena(v)); + + if (cfl_container_move_variant_to_array(v->data.as_array, replacement) != 0) { + cfl_variant_destroy(replacement); + return luaL_error(l, "cannot attach array value"); + } + + old = v->data.as_array->entries[position]; + v->data.as_array->entries[position] = replacement; + if (root->inplace) { + undo_add(root->ctx, UNDO_SET_INDEX, NULL, NULL, NULL, v->data.as_array, position, old); + } + else { + credit_value(l, root, fts_value_cost(old, 0)); + cfl_variant_destroy(old); + } + } + else { + return luaL_error(l, "set() requires a map or array"); + } + + return 0; +} + +static int value_remove(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_kvpair *pair; + struct cfl_list *head; + struct cfl_list *next; + struct cfl_list *prev; + struct fts_value *owner; + const char *key; + size_t length; + int removed = FLB_FALSE; + + v = writable_value(l); + if (v->type != CFL_VARIANT_KVLIST || lua_type(l, 2) != LUA_TSTRING) { + return luaL_error(l, "remove() requires a map and string key"); + } + + key = lua_tolstring(l, 2, &length); + owner = root_value(l, 1); + + cfl_list_foreach_safe(head, next, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (cfl_sds_len(pair->key) == length && memcmp(pair->key, key, length) == 0) { + if (owner->inplace) { + undo_reserve(l, owner->ctx); + prev = pair->_head.prev; + cfl_list_del(&pair->_head); + undo_add(owner->ctx, UNDO_REMOVE_PAIR, &v->data.as_kvlist->list, + pair, prev, NULL, 0, NULL); + } + else { + credit_value(l, owner, cost_add(128 + length, fts_value_cost(pair->val, 0))); + cfl_kvpair_destroy(pair); + } + removed = FLB_TRUE; + } + } + + lua_pushboolean(l, removed); + return 1; +} + +static int value_ensure_map(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_variant *child; + struct cfl_kvlist *map; + struct cfl_kvpair *pair; + struct fts_value *root; + const char *key; + size_t length; + + v = fts_resolve_value(l, 1, 0); + if (v->type != CFL_VARIANT_KVLIST || lua_type(l, 2) != LUA_TSTRING) { + return luaL_error(l, "ensure_map() requires a map and string key"); + } + + key = lua_tolstring(l, 2, &length); + pair = fts_find_pair(v, key, length); + if (pair && pair->val->type != CFL_VARIANT_KVLIST) { + return luaL_error(l, "existing field is not a map"); + } + + if (!pair) { + v = writable_value(l); + root = root_value(l, 1); + root->depth_changed = FLB_TRUE; + if (root->inplace) { + undo_reserve(l, root->ctx); + } + + fts_charge_value(l, root, 384 + length); + + map = cfl_kvlist_create_in(container_arena(v)); + child = map ? cfl_variant_create_from_kvlist_in(container_arena(v), map) : NULL; + if (!child) { + cfl_kvlist_destroy(map); + return luaL_error(l, "cannot allocate map"); + } + + if (cfl_kvlist_insert_s(v->data.as_kvlist, (char *) key, length, child) != 0) { + cfl_variant_destroy(child); + return luaL_error(l, "cannot insert map"); + } + + if (root->inplace) { + undo_add(root->ctx, UNDO_INSERT_PAIR, NULL, + cfl_list_entry_last(&v->data.as_kvlist->list, struct cfl_kvpair, _head), + NULL, NULL, 0, NULL); + } + } + + push_view(l, 1, 2); + return 1; +} + +static int value_len(lua_State *l) +{ + struct cfl_variant *v; + + v = fts_resolve_value(l, 1, 0); + if (v->type == CFL_VARIANT_ARRAY) { + lua_pushnumber(l, v->data.as_array->entry_count); + } + else if (v->type == CFL_VARIANT_KVLIST) { + lua_pushnumber(l, cfl_kvlist_count(v->data.as_kvlist)); + } + else { + return luaL_error(l, "len() requires a map or array"); + } + + return 1; +} + +static int value_keys(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_list *head; + struct cfl_kvpair *pair; + int i = 1; + + v = fts_resolve_value(l, 1, 0); + if (v->type != CFL_VARIANT_KVLIST) { + return luaL_error(l, "keys() requires a map"); + } + + lua_newtable(l); + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + lua_pushlstring(l, pair->key, cfl_sds_len(pair->key)); + lua_rawseti(l, -2, i++); + } + + return 1; +} + +static int value_append(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_variant *child; + struct fts_value *root; + + v = writable_value(l); + root = root_value(l, 1); + root->depth_changed = FLB_TRUE; + if (v->type != CFL_VARIANT_ARRAY) { + return luaL_error(l, "append() requires an array"); + } + + if (root->inplace) { + undo_reserve(l, root->ctx); + } + + charge_mutation(l, 2, 0); + child = from_lua(l, 2, container_arena(v)); + cfl_array_resizable(v->data.as_array, CFL_TRUE); + if (cfl_array_append(v->data.as_array, child) != 0) { + cfl_variant_destroy(child); + return luaL_error(l, "cannot append array value"); + } + + if (root->inplace) { + undo_add(root->ctx, UNDO_APPEND, NULL, NULL, NULL, v->data.as_array, 0, NULL); + } + + return 0; +} + +static int new_integer(lua_State *l) +{ + const char *str; + char *end; + size_t length; + size_t i; + int is_unsigned; + int64_t signed_value; + uint64_t unsigned_value; + lua_Number number; + struct cfl_variant *v; + struct fts_value *owner; + + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, 256); + is_unsigned = lua_toboolean(l, lua_upvalueindex(1)); + + if (lua_type(l, 1) == LUA_TNUMBER) { + number = lua_tonumber(l, 1); + if (!isfinite(number) || number != floor(number) || fabs(number) > SAFE_INTEGER || + (is_unsigned && number < 0)) { + return luaL_error(l, "use a decimal string for integers outside the safe Lua range"); + } + + if (is_unsigned) { + v = cfl_variant_create_from_uint64((uint64_t) number); + } + else { + v = cfl_variant_create_from_int64((int64_t) number); + } + } + else { + str = luaL_checklstring(l, 1, &length); + i = (!is_unsigned && length > 0 && str[0] == '-') ? 1 : 0; + if (i == length) { + return luaL_error(l, "expected a decimal integer"); + } + + for (; i < length; i++) { + if (str[i] < '0' || str[i] > '9') { + return luaL_error(l, "expected a decimal integer"); + } + } + + errno = 0; + if (is_unsigned) { + unsigned_value = strtoull(str, &end, 10); + v = errno == ERANGE ? NULL : cfl_variant_create_from_uint64(unsigned_value); + } + else { + signed_value = strtoll(str, &end, 10); + v = errno == ERANGE ? NULL : cfl_variant_create_from_int64(signed_value); + } + + if (errno == ERANGE) { + return luaL_error(l, "integer outside 64-bit range"); + } + } + + if (!v) { + return luaL_error(l, "cannot allocate integer"); + } + + owner->owned = v; + return 1; +} + +static int new_container(lua_State *l) +{ + struct cfl_variant *v; + struct fts_value *owner; + struct cfl_kvlist *map; + struct cfl_array *array; + + if (lua_gettop(l) != 0) { + return luaL_error(l, "map()/array() create empty containers and take no arguments"); + } + + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, 256); + + if (lua_toboolean(l, lua_upvalueindex(1))) { + map = cfl_kvlist_create(); + v = map ? cfl_variant_create_from_kvlist(map) : NULL; + if (!v) { + cfl_kvlist_destroy(map); + } + } + else { + array = cfl_array_create(0); + v = array ? cfl_variant_create_from_array(array) : NULL; + if (!v) { + cfl_array_destroy(array); + } + } + + if (!v) { + return luaL_error(l, "cannot allocate container"); + } + + owner->owned = v; + return 1; +} + +static int new_bytes(lua_State *l) +{ + const char *str; + size_t length; + struct cfl_variant *v; + struct fts_value *owner; + + if (lua_type(l, 1) != LUA_TSTRING) { + return luaL_error(l, "bytes() requires a Lua string"); + } + + str = lua_tolstring(l, 1, &length); + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, length + 256); + + v = cfl_variant_create_from_bytes((char *) str, length, CFL_FALSE); + if (!v) { + return luaL_error(l, "cannot allocate bytes"); + } + + owner->owned = v; + return 1; +} + +static void push_folded_key(lua_State *l, const char *key, size_t length) +{ + luaL_Buffer buffer; + size_t index; + + luaL_buffinit(l, &buffer); + for (index = 0; index < length; index++) { + luaL_addchar(&buffer, tolower((unsigned char) key[index])); + } + luaL_pushresult(&buffer); +} + +/* Iterate the stored pairs, including duplicate parent keys. Named lookup + * cannot visit all occurrences and is unsuitable for privacy operations. */ +static void redact_value(lua_State *l, struct cfl_variant *value, int depth, int keys) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_kvpair *pair; + struct cfl_variant *replacement; + struct cfl_variant *old; + struct cfl_list *previous; + struct fts_runtime *ctx; + struct fts_value *owner; + void *data; + size_t length; + size_t i; + int match; + int seen; + int duplicate; + + lua_getallocf(l, &data); + ctx = data; + if (depth >= FTS_MAX_DEPTH || (!ctx->jit && --ctx->instructions_left <= 0)) { + luaL_error(l, "redaction nesting or instruction limit exceeded"); + } + + if (!lua_checkstack(l, 8)) { + luaL_error(l, "redaction stack limit exceeded"); + } + + owner = root_value(l, 1); + + if (value->type == CFL_VARIANT_ARRAY) { + for (i = 0; i < value->data.as_array->entry_count; i++) { + redact_value(l, value->data.as_array->entries[i], depth + 1, keys); + } + } + else if (value->type == CFL_VARIANT_KVLIST) { + lua_newtable(l); + seen = lua_gettop(l); + + cfl_list_foreach_safe(head, tmp, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + length = cfl_sds_len(pair->key); + + push_folded_key(l, pair->key, length); + lua_rawget(l, keys); + match = lua_toboolean(l, -1); + lua_pop(l, 1); + + if (match) { + lua_pushlstring(l, pair->key, length); + lua_rawget(l, seen); + duplicate = lua_toboolean(l, -1); + lua_pop(l, 1); + + if (duplicate) { + if (owner->inplace) { + undo_reserve(l, owner->ctx); + previous = pair->_head.prev; + cfl_list_del(&pair->_head); + undo_add(owner->ctx, UNDO_REMOVE_PAIR, &value->data.as_kvlist->list, + pair, previous, NULL, 0, NULL); + } + else { + credit_value(l, owner, cost_add(128 + length, + fts_value_cost(pair->val, 0))); + cfl_kvpair_destroy(pair); + } + continue; + } + + lua_pushlstring(l, pair->key, length); + lua_pushboolean(l, 1); + lua_rawset(l, seen); + + if (owner->inplace) { + undo_reserve(l, owner->ctx); + } + + charge_mutation(l, 3, 0); + replacement = from_lua(l, 3, container_arena(value)); + if (cfl_container_move_variant_to_kvlist(value->data.as_kvlist, + replacement) != 0) { + cfl_variant_destroy(replacement); + luaL_error(l, "cannot attach redacted value"); + } + + old = pair->val; + pair->val = replacement; + if (owner->inplace) { + undo_add(owner->ctx, UNDO_SET_PAIR, NULL, pair, NULL, NULL, 0, old); + } + else { + credit_value(l, owner, fts_value_cost(old, 0)); + cfl_variant_destroy(old); + } + } + else { + redact_value(l, pair->val, depth + 1, keys); + } + } + + lua_pop(l, 1); + } +} + +static int redact(lua_State *l) +{ + struct cfl_variant *value; + const char *key; + size_t length; + size_t index; + size_t count; + int keys; + + luaL_checktype(l, 2, LUA_TTABLE); + if (lua_isnoneornil(l, 3)) { + lua_settop(l, 2); + lua_pushliteral(l, "[REDACTED]"); + } + + value = writable_value(l); + if (lua_type(l, 3) == LUA_TUSERDATA) { + root_value(l, 1)->depth_changed = FLB_TRUE; + } + + count = lua_objlen(l, 2); + lua_newtable(l); + keys = lua_gettop(l); + + for (index = 1; index <= count; index++) { + lua_rawgeti(l, 2, index); + luaL_checktype(l, -1, LUA_TSTRING); + key = lua_tolstring(l, -1, &length); + push_folded_key(l, key, length); + lua_pushboolean(l, 1); + lua_rawset(l, keys); + lua_pop(l, 1); + } + + redact_value(l, value, 0, keys); + return 0; +} + +/* An owned, writable copy that may outlive the callback. */ +static int value_copy(lua_State *l) +{ + struct cfl_variant *source; + struct fts_value *value; + size_t cost; + + source = fts_resolve_value(l, 1, 0); + cost = fts_value_cost(source, 0); + if (cost == SIZE_MAX) { + return luaL_error(l, "cannot copy typed value (allocation or nesting limit)"); + } + + value = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, value, cost); + + value->owned = fts_copy_value(source, 0); + if (!value->owned) { + return luaL_error(l, "cannot copy typed value (allocation or nesting limit)"); + } + + return 1; +} + +static const luaL_Reg value_methods[] = { + {"__gc", value_gc}, + {"type", value_type}, + {"value", value_scalar}, + {"number", value_number}, + {"get", value_get}, + {"get_string", value_get_string}, + {"get_bool", value_get_bool}, + {"get_number", value_get_number}, + {"set", value_set}, + {"remove", value_remove}, + {"ensure_map", value_ensure_map}, + {"len", value_len}, + {"keys", value_keys}, + {"append", value_append}, + {"copy", value_copy}, + {NULL, NULL} +}; + +struct json_key_slot { + yyjson_val *key; + uint64_t hash; +}; + +static size_t json_value_cost(yyjson_val *value, int depth) +{ + yyjson_val *key; + yyjson_val *item; + size_t index; + size_t count; + size_t cost = 256; + size_t child; + + if (depth >= FTS_MAX_DEPTH) { + return SIZE_MAX; + } + + if (yyjson_is_str(value)) { + return cost + yyjson_get_len(value); + } + + if (yyjson_is_obj(value)) { + yyjson_obj_foreach(value, index, count, key, item) { + child = json_value_cost(item, depth + 1); + if (child > SIZE_MAX - cost || + yyjson_get_len(key) + 128 > SIZE_MAX - cost - child) { + return SIZE_MAX; + } + cost += child + yyjson_get_len(key) + 128; + } + } + else if (yyjson_is_arr(value)) { + yyjson_arr_foreach(value, index, count, item) { + child = json_value_cost(item, depth + 1); + if (child > SIZE_MAX - cost) { + return SIZE_MAX; + } + cost += child; + } + } + + return cost; +} + +/* Strict JSON conversion preserves integer widths and rejects duplicate keys. */ +static struct cfl_variant *json_value(yyjson_val *value, int depth, struct fts_runtime *ctx) +{ + struct cfl_variant *out; + struct cfl_variant *child; + struct cfl_array *array; + struct cfl_kvlist *map; + yyjson_val *key; + yyjson_val *item; + size_t index; + size_t count; + size_t capacity; + size_t slot; + uint64_t hash; + struct json_key_slot *keys; + + if (depth >= FTS_MAX_DEPTH || (!ctx->jit && --ctx->instructions_left <= 0)) { + return NULL; + } + + if (yyjson_is_obj(value)) { + map = cfl_kvlist_create(); + out = map ? cfl_variant_create_from_kvlist(map) : NULL; + if (!out) { + cfl_kvlist_destroy(map); + return NULL; + } + + count = yyjson_obj_size(value); + capacity = 1; + while (capacity < count * 2) { + capacity *= 2; + } + + keys = calloc(capacity, sizeof(*keys)); + if (keys == NULL) { + flb_errno(); + cfl_variant_destroy(out); + return NULL; + } + + yyjson_obj_foreach(value, index, count, key, item) { + hash = cfl_hash_64bits(yyjson_get_str(key), yyjson_get_len(key)); + slot = hash & (capacity - 1); + while (keys[slot].key != NULL) { + if ((!ctx->jit && --ctx->instructions_left <= 0) || + (keys[slot].hash == hash && + yyjson_get_len(keys[slot].key) == yyjson_get_len(key) && + memcmp(yyjson_get_str(keys[slot].key), yyjson_get_str(key), + yyjson_get_len(key)) == 0)) { + free(keys); + cfl_variant_destroy(out); + return NULL; + } + slot = (slot + 1) & (capacity - 1); + } + + keys[slot].key = key; + keys[slot].hash = hash; + + child = json_value(item, depth + 1, ctx); + if (!child || cfl_kvlist_insert_s(map, (char *) yyjson_get_str(key), + yyjson_get_len(key), child) != 0) { + free(keys); + cfl_variant_destroy(child); + cfl_variant_destroy(out); + return NULL; + } + } + + free(keys); + return out; + } + + if (yyjson_is_arr(value)) { + array = cfl_array_create(yyjson_arr_size(value)); + out = array ? cfl_variant_create_from_array(array) : NULL; + if (!out) { + cfl_array_destroy(array); + return NULL; + } + + yyjson_arr_foreach(value, index, count, item) { + child = json_value(item, depth + 1, ctx); + if (!child || cfl_array_append(array, child) != 0) { + cfl_variant_destroy(child); + cfl_variant_destroy(out); + return NULL; + } + } + + return out; + } + + if (yyjson_is_str(value)) { + return cfl_variant_create_from_string_s((char *) yyjson_get_str(value), + yyjson_get_len(value), CFL_FALSE); + } + + if (yyjson_is_uint(value)) { + return cfl_variant_create_from_uint64(yyjson_get_uint(value)); + } + + if (yyjson_is_sint(value)) { + return cfl_variant_create_from_int64(yyjson_get_sint(value)); + } + + if (yyjson_is_real(value)) { + return cfl_variant_create_from_double(yyjson_get_real(value)); + } + + if (yyjson_is_bool(value)) { + return cfl_variant_create_from_bool(yyjson_get_bool(value)); + } + + if (yyjson_is_null(value)) { + return cfl_variant_create_from_null(); + } + + return NULL; +} + +static int parse_json(lua_State *l) +{ + struct fts_value *owner; + struct fts_runtime *ctx; + yyjson_doc *doc; + const char *text; + size_t length; + size_t reservation; + size_t cost; + + luaL_checktype(l, 1, LUA_TSTRING); + text = lua_tolstring(l, 1, &length); + if (length > 1048576) { + return luaL_error(l, "JSON input exceeds 1 MiB"); + } + + owner = fts_push_owned(l, NULL, FLB_FALSE); + ctx = owner->ctx; + + /* Bound the parser's temporary document separately from the owned value. */ + reservation = length * 16 + 256; + fts_charge_value(l, owner, reservation); + + doc = yyjson_read(text, length, YYJSON_READ_BIGNUM_AS_RAW); + if (!doc) { + return luaL_error(l, "invalid JSON"); + } + + cost = json_value_cost(yyjson_doc_get_root(doc), 0); + if (cost > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - cost || + ctx->lua_bytes > ctx->memory_limit - ctx->native_bytes - cost) { + lua_gc(l, LUA_GCCOLLECT, 0); + if (cost > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - cost || + ctx->lua_bytes > ctx->memory_limit - ctx->native_bytes - cost) { + yyjson_doc_free(doc); + return luaL_error(l, "JSON value exceeds policy memory or nesting limit"); + } + } + + fts_charge_value(l, owner, cost); + owner->owned = json_value(yyjson_doc_get_root(doc), 0, ctx); + yyjson_doc_free(doc); + credit_value(l, owner, reservation); + + if (!owner->owned) { + return luaL_error(l, + "JSON exceeds type/depth/instruction limits or contains duplicate keys"); + } + + return 1; +} + +/* Exact aggregation never rounds integer telemetry through lua_Number. */ +static int add_metric_values(lua_State *l) +{ + struct cfl_variant *left; + struct cfl_variant *right; + struct fts_value *owner; + int64_t a; + int64_t b; + uint64_t unsigned_value; + uint64_t magnitude; + uint64_t total; + double x; + double y; + + left = fts_resolve_value(l, 1, 0); + right = fts_resolve_value(l, 2, 0); + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, 256); + + if (left->type == CFL_VARIANT_INT && right->type == CFL_VARIANT_INT) { + a = left->data.as_int64; + b = right->data.as_int64; + if ((b > 0 && a > INT64_MAX - b) || (b < 0 && a < INT64_MIN - b)) { + return luaL_error(l, "metric integer aggregation overflow"); + } + owner->owned = cfl_variant_create_from_int64(a + b); + } + else if (left->type == CFL_VARIANT_UINT && right->type == CFL_VARIANT_UINT) { + if (left->data.as_uint64 > UINT64_MAX - right->data.as_uint64) { + return luaL_error(l, "metric count aggregation overflow"); + } + owner->owned = cfl_variant_create_from_uint64(left->data.as_uint64 + + right->data.as_uint64); + } + else if ((left->type == CFL_VARIANT_INT && right->type == CFL_VARIANT_UINT) || + (left->type == CFL_VARIANT_UINT && right->type == CFL_VARIANT_INT)) { + if (left->type == CFL_VARIANT_INT) { + a = left->data.as_int64; + } + else { + a = right->data.as_int64; + } + + if (left->type == CFL_VARIANT_UINT) { + unsigned_value = left->data.as_uint64; + } + else { + unsigned_value = right->data.as_uint64; + } + + if (a < 0) { + magnitude = (uint64_t) -(a + 1) + 1; + if (unsigned_value < magnitude) { + magnitude -= unsigned_value; + if (magnitude == (uint64_t) INT64_MAX + 1) { + owner->owned = cfl_variant_create_from_int64(INT64_MIN); + } + else { + owner->owned = cfl_variant_create_from_int64(-(int64_t) magnitude); + } + } + else { + total = unsigned_value - magnitude; + if (total <= INT64_MAX) { + owner->owned = cfl_variant_create_from_int64((int64_t) total); + } + else { + owner->owned = cfl_variant_create_from_uint64(total); + } + } + } + else { + if (unsigned_value > UINT64_MAX - (uint64_t) a) { + return luaL_error(l, "metric integer aggregation overflow"); + } + + total = unsigned_value + (uint64_t) a; + if (total <= INT64_MAX) { + owner->owned = cfl_variant_create_from_int64((int64_t) total); + } + else { + owner->owned = cfl_variant_create_from_uint64(total); + } + } + } + else { + if ((left->type != CFL_VARIANT_INT && left->type != CFL_VARIANT_UINT && + left->type != CFL_VARIANT_DOUBLE) || + (right->type != CFL_VARIANT_INT && right->type != CFL_VARIANT_UINT && + right->type != CFL_VARIANT_DOUBLE)) { + return luaL_error(l, "metric aggregation requires numeric values"); + } + + if (left->type == CFL_VARIANT_DOUBLE) { + x = left->data.as_double; + } + else if (left->type == CFL_VARIANT_INT) { + x = (double) left->data.as_int64; + } + else { + x = (double) left->data.as_uint64; + } + + if (right->type == CFL_VARIANT_DOUBLE) { + y = right->data.as_double; + } + else if (right->type == CFL_VARIANT_INT) { + y = (double) right->data.as_int64; + } + else { + y = (double) right->data.as_uint64; + } + + if ((left->type != CFL_VARIANT_DOUBLE && fabs(x) > SAFE_INTEGER) || + (right->type != CFL_VARIANT_DOUBLE && fabs(y) > SAFE_INTEGER) || !isfinite(x + y)) { + return luaL_error(l, "metric aggregation would lose precision or overflow"); + } + owner->owned = cfl_variant_create_from_double(x + y); + } + + if (!owner->owned) { + return luaL_error(l, "cannot allocate aggregated value"); + } + + return 1; +} + +int fts_register_values(lua_State *l) +{ + struct fts_value *null_owner; + struct fts_runtime *ctx; + int index; + static const char *keys[] = {"tag", "body", "metadata", "timestamp", "sec", "nsec", + "context", "attributes"}; + + lua_getallocf(l, (void **) &ctx); + for (index = 0; index < FTS_KEY_COUNT; index++) { + lua_pushstring(l, keys[index]); + ctx->key_refs[index] = luaL_ref(l, LUA_REGISTRYINDEX); + } + + lua_pushliteral(l, ""); + ctx->tag_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + /* Value metatable: finalizer plus the shared method table. */ + luaL_newmetatable(l, VALUE_MT); + ctx->value_metatable = lua_topointer(l, -1); + lua_pushvalue(l, -1); + ctx->value_metatable_ref = luaL_ref(l, LUA_REGISTRYINDEX); + luaL_register(l, NULL, value_methods); + lua_newtable(l); + luaL_register(l, NULL, value_methods + 1); + lua_setfield(l, -2, "__index"); + lua_pushliteral(l, "typed value"); + lua_setfield(l, -2, "__metatable"); + + /* Finalizer-free metatable for borrowed values and views. */ + lua_newtable(l); + lua_getfield(l, -2, "__index"); + lua_setfield(l, -2, "__index"); + lua_pushliteral(l, "typed value"); + lua_setfield(l, -2, "__metatable"); + ctx->empty_metatable = lua_topointer(l, -1); + ctx->empty_metatable_ref = luaL_ref(l, LUA_REGISTRYINDEX); + lua_pop(l, 1); + + /* package.loaded.fluentbit */ + lua_getglobal(l, "package"); + lua_getfield(l, -1, "loaded"); + lua_newtable(l); + + lua_pushboolean(l, 0); + lua_pushcclosure(l, new_integer, 1); + lua_setfield(l, -2, "int64"); + + lua_pushboolean(l, 1); + lua_pushcclosure(l, new_integer, 1); + lua_setfield(l, -2, "uint64"); + + lua_pushboolean(l, 1); + lua_pushcclosure(l, new_container, 1); + lua_setfield(l, -2, "map"); + + lua_pushboolean(l, 0); + lua_pushcclosure(l, new_container, 1); + lua_setfield(l, -2, "array"); + + lua_pushcfunction(l, equal_values); + lua_setfield(l, -2, "_fts_equal"); + lua_pushcfunction(l, add_metric_values); + lua_setfield(l, -2, "_fts_add"); + lua_pushcfunction(l, redact); + lua_setfield(l, -2, "redact"); + lua_pushcfunction(l, parse_json); + lua_setfield(l, -2, "parse_json"); + lua_pushcfunction(l, new_bytes); + lua_setfield(l, -2, "bytes"); + + null_owner = fts_push_owned(l, NULL, FLB_TRUE); + fts_charge_value(l, null_owner, 256); + null_owner->owned = cfl_variant_create_from_null(); + if (!null_owner->owned) { + lua_pop(l, 3); + return -1; + } + lua_setfield(l, -2, "null"); + + lua_pushliteral(l, "keep"); + lua_setfield(l, -2, "KEEP"); + lua_pushliteral(l, "drop"); + lua_setfield(l, -2, "DROP"); + + lua_setfield(l, -2, "fluentbit"); + lua_pop(l, 2); + + if (fts_load_helpers(l, fts_value_helpers, sizeof(fts_value_helpers) - 1, + "fluentbit.helpers") != 0) { + return -1; + } + + return 0; +} + +int fts_valid_utf8(const char *text, size_t length) +{ + const unsigned char *bytes = (const unsigned char *) text; + size_t i = 0; + unsigned char first; + unsigned char second; + int count; + int j; + + while (i < length) { + first = bytes[i++]; + if (first < 0x80) { + continue; + } + + if (first >= 0xc2 && first <= 0xdf) { + count = 1; + } + else if (first >= 0xe0 && first <= 0xef) { + count = 2; + } + else if (first >= 0xf0 && first <= 0xf4) { + count = 3; + } + else { + count = 0; + } + + if (!count || length - i < (size_t) count) { + return FLB_FALSE; + } + + second = bytes[i]; + if ((first == 0xe0 && second < 0xa0) || (first == 0xed && second > 0x9f) || + (first == 0xf0 && second < 0x90) || (first == 0xf4 && second > 0x8f)) { + return FLB_FALSE; + } + + for (j = 0; j < count; j++) { + if (bytes[i] < 0x80 || bytes[i++] > 0xbf) { + return FLB_FALSE; + } + } + } + + return FLB_TRUE; +} From ae98413372e18d6548784d0f9a2ad7cde2b64fdb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 14/61] tests: internal: mp: cover record pooling and the log scanner Signed-off-by: Eduardo Silva --- tests/internal/mp.c | 68 ++++++++++++ tests/internal/mp_chunk_cobj.c | 189 +++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) diff --git a/tests/internal/mp.c b/tests/internal/mp.c index 66d9ab3f14f..ee556122f99 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include @@ -742,7 +744,73 @@ static void test_preflight_formats(void) flb_mp_preflight_destroy(scan); } +/* Compare optimized counts with the authoritative decoder, including every + * truncated prefix and byte corruption of grouped and ordinary records. */ +static int reference_log_count(const char *data, size_t size) +{ + struct flb_log_event_decoder decoder; + struct flb_log_event event; + int count = 0; + + if (flb_log_event_decoder_init(&decoder, (char *) data, size) != 0) { + return 0; + } + while (flb_log_event_decoder_next(&decoder, &event) == 0) { + count++; + } + flb_log_event_decoder_destroy(&decoder); + return count; +} + +static void test_log_scan_matches_decoder(void) +{ + struct flb_log_event_encoder *encoder; + struct flb_time timestamp; + size_t length; + size_t index; + int ret; + int count; + int plain; + int record; + char saved; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (record = 0; record < 4; record++) { + if (record == 1) { + TEST_ASSERT(flb_log_event_encoder_group_init(encoder) == 0); + TEST_ASSERT(flb_log_event_encoder_group_header_end(encoder) == 0); + } + TEST_ASSERT(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(×tamp, 1700000000 + record, record); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(encoder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("key"), FLB_LOG_EVENT_CSTRING_VALUE("value")); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(encoder) == 0); + if (record == 2) { + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + } + } + TEST_CHECK(flb_mp_scan_log_records(encoder->output_buffer, encoder->output_length, + FLB_TRUE, &count, &plain) == 0); + TEST_CHECK(count == 4 && plain == FLB_FALSE); + for (length = 0; length <= encoder->output_length; length++) { + TEST_CHECK(flb_mp_count_log_records(encoder->output_buffer, length) == + reference_log_count(encoder->output_buffer, length)); + } + for (index = 0; index < encoder->output_length; index++) { + saved = encoder->output_buffer[index]; + encoder->output_buffer[index] = (char) 0xc1; + TEST_CHECK(flb_mp_count_log_records(encoder->output_buffer, encoder->output_length) == + reference_log_count(encoder->output_buffer, encoder->output_length)); + encoder->output_buffer[index] = saved; + } + flb_log_event_encoder_destroy(encoder); +} + TEST_LIST = { + {"log_scan_matches_decoder", test_log_scan_matches_decoder}, {"preflight", test_preflight}, {"preflight_formats", test_preflight_formats}, {"validate_trace_chunk" , test_validate_trace_chunk}, diff --git a/tests/internal/mp_chunk_cobj.c b/tests/internal/mp_chunk_cobj.c index e10be583482..eb7e93f1242 100644 --- a/tests/internal/mp_chunk_cobj.c +++ b/tests/internal/mp_chunk_cobj.c @@ -26,6 +26,7 @@ #include #include +#include #include @@ -553,7 +554,195 @@ void record_next_condition_filter() } +static void bounded_record_pool_preserves_moved_records(void) +{ + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder = {0}; + struct flb_mp_chunk_cobj *source; + struct flb_mp_chunk_cobj *destination; + struct flb_mp_chunk_record *records[1024]; + int moved[] = {0, 511, 512, 1023}; + size_t index; + size_t pooled = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + source = flb_mp_chunk_cobj_create(encoder, &decoder); + destination = flb_mp_chunk_cobj_create(encoder, &decoder); + TEST_ASSERT(source != NULL && destination != NULL); + source->record_pool_enabled = FLB_TRUE; + for (index = 0; index < 1024; index++) { + records[index] = flb_mp_chunk_record_create(source); + TEST_ASSERT(records[index] != NULL); + TEST_CHECK(records[index]->cobj_record == NULL); + TEST_CHECK(records[index]->owns_group_attributes == FLB_FALSE); + pooled += records[index]->storage_block != NULL; + records[index]->modified = index; + cfl_list_add(&records[index]->_head, &source->records); + } + TEST_CHECK(pooled == 512); + TEST_CHECK(source->record_pool_blocks == 8); + for (index = 0; index < sizeof(moved) / sizeof(moved[0]); index++) { + cfl_list_del(&records[moved[index]]->_head); + cfl_list_add(&records[moved[index]]->_head, &destination->records); + } + TEST_CHECK(flb_mp_chunk_cobj_destroy(source) == 0); + for (index = 0; index < sizeof(moved) / sizeof(moved[0]); index++) { + TEST_CHECK(records[moved[index]]->modified == moved[index]); + } + TEST_CHECK(flb_mp_chunk_cobj_destroy(destination) == 0); + flb_log_event_encoder_destroy(encoder); +} + +static void scratch_encode_preserves_field_boundaries(void) +{ + struct flb_log_event_encoder *builder; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event_decoder verify; + struct flb_log_event event; + struct flb_mp_chunk_cobj *chunk; + struct flb_mp_chunk_record *record; + struct flb_time timestamp; + char payload[9000]; + size_t lengths[] = {0, 1, sizeof(payload), 2}; + size_t length; + size_t index; + char *output; + size_t output_size; + int ret; + + memset(payload, 0xff, sizeof(payload)); + payload[1] = '\0'; + builder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(builder != NULL && encoder != NULL); + flb_time_set(×tamp, 1, 0); + for (index = 0; index < 16; index++) { + length = lengths[index % 4]; + TEST_ASSERT(flb_log_event_encoder_begin_record(builder) == 0); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(builder, ×tamp) == 0); + if (index % 2) { + ret = flb_log_event_encoder_append_metadata_values(builder, + FLB_LOG_EVENT_CSTRING_VALUE("text"), FLB_LOG_EVENT_STRING_VALUE(payload, length)); + TEST_ASSERT(ret == 0); + } + ret = flb_log_event_encoder_append_body_values(builder, + FLB_LOG_EVENT_CSTRING_VALUE("binary"), FLB_LOG_EVENT_BINARY_VALUE(payload, length)); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(builder) == 0); + } + TEST_ASSERT(flb_log_event_decoder_init(&decoder, builder->output_buffer, + builder->output_length) == 0); + chunk = flb_mp_chunk_cobj_create(encoder, &decoder); + TEST_ASSERT(chunk != NULL); + flb_mp_chunk_cobj_value_pool_enable(chunk); + TEST_ASSERT(chunk->value_pool != NULL); + while ((ret = flb_mp_chunk_cobj_record_next(chunk, &record)) == FLB_MP_CHUNK_RECORD_OK) { + TEST_CHECK(record->cobj_record->variant->arena == chunk->value_pool); + } + TEST_ASSERT(ret == FLB_MP_CHUNK_RECORD_EOF); + TEST_ASSERT(flb_mp_chunk_cobj_encode(chunk, &output, &output_size) == 0); + TEST_ASSERT(flb_log_event_decoder_init(&verify, output, output_size) == 0); + index = 0; + while (flb_log_event_decoder_next(&verify, &event) == FLB_EVENT_DECODER_SUCCESS) { + length = lengths[index % 4]; + TEST_ASSERT(event.body->type == MSGPACK_OBJECT_MAP && event.body->via.map.size == 1); + TEST_ASSERT(event.body->via.map.ptr[0].val.type == MSGPACK_OBJECT_BIN); + TEST_CHECK(event.body->via.map.ptr[0].val.via.bin.size == length); + TEST_CHECK(memcmp(event.body->via.map.ptr[0].val.via.bin.ptr, payload, length) == 0); + TEST_ASSERT(event.metadata->type == MSGPACK_OBJECT_MAP); + TEST_CHECK(event.metadata->via.map.size == index % 2); + if (index % 2) { + TEST_ASSERT(event.metadata->via.map.ptr[0].val.type == MSGPACK_OBJECT_STR); + TEST_CHECK(event.metadata->via.map.ptr[0].val.via.str.size == length); + TEST_CHECK(memcmp(event.metadata->via.map.ptr[0].val.via.str.ptr, payload, length) == 0); + } + index++; + } + TEST_CHECK(index == 16); + flb_log_event_decoder_destroy(&verify); + flb_mp_chunk_cobj_destroy(chunk); + flb_log_event_decoder_destroy(&decoder); + flb_log_event_encoder_destroy(builder); + flb_log_event_encoder_destroy(encoder); + flb_free(output); +} + +static void scratch_encode_failure_releases_storage(void) +{ + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder = {0}; + struct flb_mp_chunk_cobj *chunk; + struct flb_mp_chunk_record *record; + char *output = NULL; + size_t output_size = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + chunk = flb_mp_chunk_cobj_create(encoder, &decoder); + TEST_ASSERT(chunk != NULL); + record = flb_mp_chunk_record_create(chunk); + TEST_ASSERT(record != NULL); + record->cobj_record = cfl_object_create(); + TEST_ASSERT(record->cobj_record != NULL); + cfl_list_add(&record->_head, &chunk->records); + TEST_CHECK(flb_mp_chunk_cobj_encode(chunk, &output, &output_size) == -1); + TEST_CHECK(output == NULL && output_size == 0); + flb_mp_chunk_cobj_destroy(chunk); + flb_log_event_encoder_destroy(encoder); +} + +static void exhausted_value_pool_falls_back_to_heap(void) +{ + struct flb_log_event_encoder *builder; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_mp_chunk_cobj *chunk; + struct flb_mp_chunk_record *record; + struct flb_time timestamp; + size_t blocks = 0; + int ret; + + builder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(builder != NULL && encoder != NULL); + flb_time_set(×tamp, 1, 0); + TEST_ASSERT(flb_log_event_encoder_begin_record(builder) == 0); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(builder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(builder, + FLB_LOG_EVENT_CSTRING_VALUE("message"), FLB_LOG_EVENT_CSTRING_VALUE("preserved")); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(builder) == 0); + TEST_ASSERT(flb_log_event_decoder_init(&decoder, builder->output_buffer, + builder->output_length) == 0); + chunk = flb_mp_chunk_cobj_create(encoder, &decoder); + TEST_ASSERT(chunk != NULL); + flb_mp_chunk_cobj_value_pool_enable(chunk); + TEST_ASSERT(chunk->value_pool != NULL); + TEST_CHECK(flb_mp_chunk_cobj_value_pool_get(chunk, SIZE_MAX) == NULL); + /* Trees already in the pool can grow past the budget; new trees cannot */ + while (blocks < 80 && cfl_arena_malloc(chunk->value_pool, 16384) != NULL) { + blocks++; + } + TEST_CHECK(blocks == 80); + TEST_CHECK(chunk->value_pool_bytes > 1024 * 1024); + TEST_CHECK(flb_mp_chunk_cobj_value_pool_get(chunk, 512) == NULL); + TEST_ASSERT(flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK); + TEST_CHECK(record->cobj_record->variant->arena == NULL); + TEST_CHECK(record->cobj_metadata->variant->arena == NULL); + TEST_CHECK(cfl_kvlist_contains(record->cobj_record->variant->data.as_kvlist, "message") == 1); + flb_mp_chunk_cobj_destroy(chunk); + flb_log_event_decoder_destroy(&decoder); + flb_log_event_encoder_destroy(builder); + flb_log_event_encoder_destroy(encoder); +} + TEST_LIST = { + { "exhausted_value_pool_heap_fallback", exhausted_value_pool_falls_back_to_heap }, + { "bounded_record_pool_moved_lifetime", bounded_record_pool_preserves_moved_records }, + { "scratch_encode_field_boundaries", scratch_encode_preserves_field_boundaries }, + { "scratch_encode_failure_cleanup", scratch_encode_failure_releases_storage }, { "decoder_groups_cobj", decoder_groups_cobj }, { "record_next_condition_filter", record_next_condition_filter }, { 0 } From 1e345d30941475d603aa00dfb520bcd8b573f018 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 15/61] tests: internal: processor: cover metrics failure output Signed-off-by: Eduardo Silva --- tests/internal/processor.c | 60 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/internal/processor.c b/tests/internal/processor.c index e2ebcd7bf86..7c6481e5029 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -625,7 +625,67 @@ static void processor_metrics_counters() flb_sds_destroy(regex_prop_key); } +static int create_intermediate_metrics(struct flb_processor_instance *ins, + struct cmt *input, struct cmt **output, + const char *tag, int tag_len) +{ + *output = cmt_create(); + return *output == NULL ? -1 : FLB_PROCESSOR_SUCCESS; +} + +static int fail_metrics_processing(struct flb_processor_instance *ins, + struct cmt *input, struct cmt **output, + const char *tag, int tag_len) +{ + *output = NULL; + return -1; +} + +static void processor_metrics_failure_clears_output(void) +{ + struct flb_config *config; + struct flb_processor *proc; + struct cmt *input; + void *output; + size_t output_size = 123; + struct flb_processor_plugin first = { + .name = "test_metrics_copy", + .description = "create intermediate metrics", + .cb_process_metrics = create_intermediate_metrics + }; + struct flb_processor_plugin second = { + .name = "test_metrics_failure", + .description = "fail metrics processing", + .cb_process_metrics = fail_metrics_processing + }; + + flb_init_env(); + config = flb_config_init(); + TEST_ASSERT(config != NULL); + mk_list_add(&first._head, &config->processor_plugins); + mk_list_add(&second._head, &config->processor_plugins); + proc = flb_processor_create(config, "failure", NULL, 0); + TEST_ASSERT(proc != NULL); + TEST_ASSERT(flb_processor_unit_create(proc, FLB_PROCESSOR_METRICS, first.name) != NULL); + TEST_ASSERT(flb_processor_unit_create(proc, FLB_PROCESSOR_METRICS, second.name) != NULL); + TEST_ASSERT(flb_processor_init(proc) == 0); + input = cmt_create(); + TEST_ASSERT(input != NULL); + output = input; + TEST_CHECK(flb_processor_run(proc, 0, FLB_PROCESSOR_METRICS, "test", 4, + input, 0, &output, &output_size) == -1); + TEST_CHECK(output == NULL); + TEST_CHECK(output_size == 0); + /* The caller still owns the original input after a failed pipeline. */ + cmt_destroy(input); + flb_processor_destroy(proc); + mk_list_del(&first._head); + mk_list_del(&second._head); + flb_config_exit(config); +} + TEST_LIST = { + {"processor_metrics_failure_clears_output", processor_metrics_failure_clears_output}, { "processor_private_inputs_use_main_loop", processor_private_inputs_use_main_loop }, { "processor", processor }, { "processor_grouped_filter_counters", processor_grouped_filter_counters }, From 056a7bb6c1c7ac9010a798d0d62d26fe8caf7493 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 16/61] tests: internal: opentelemetry: cover typed values and series merge Signed-off-by: Eduardo Silva --- tests/internal/opentelemetry.c | 475 +++++++++++++++++++++++++++++++++ 1 file changed, 475 insertions(+) diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 0967329a861..0c44e6e318e 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -44,6 +44,9 @@ #include #include #include +#include +#include +#include #include #include @@ -1353,6 +1356,15 @@ static void run_metrics_case(msgpack_object *case_obj, const char *case_name) exp_histogram = cfl_list_entry(context->exp_histograms.next, struct cmt_exp_histogram, _head); metric = &exp_histogram->map->metric; + /* JSON decoding preserves previous windows; check the latest + * point for cases that supplied several unlabeled observations. */ + if (exp_histogram->map->label_count == 0 && + !cfl_list_is_empty(&exp_histogram->map->metrics)) { + metric = cfl_list_entry(exp_histogram->map->metrics.prev, + struct cmt_metric, _head); + TEST_CHECK(cmt_metric_get_timestamp(metric) >= + cmt_metric_get_timestamp(&exp_histogram->map->metric)); + } test_check_metric_description_unit(exp_histogram_obj, &exp_histogram->opts, exp_histogram->map); @@ -3532,8 +3544,467 @@ void test_opentelemetry_metrics_cutoff() cmt_destroy(cmt); } +static void *test_find_metric_family(struct cfl_list *families, size_t head_offset, + size_t opts_offset, const char *name) +{ + struct cfl_list *head; + struct cmt_opts *opts; + char *family; + + cfl_list_foreach(head, families) { + family = ((char *) head) - head_offset; + opts = (struct cmt_opts *) (family + opts_offset); + if (opts->fqname != NULL && strcmp(opts->fqname, name) == 0) { + return family; + } + } + + return NULL; +} + +#define TEST_FIND_FAMILY(list, type, name) \ + ((type *) test_find_metric_family(list, offsetof(type, _head), offsetof(type, opts), name)) + +static struct cmt_metric *test_single_dynamic_sample(struct cmt_map *map) +{ + if (map == NULL || map->metric_static_set || cfl_list_size(&map->metrics) != 1) { + return NULL; + } + + return cfl_list_entry_first(&map->metrics, struct cmt_metric, _head); +} + +static size_t test_kvlist_key_count(struct cfl_kvlist *list, const char *key) +{ + size_t count; + struct cfl_list *head; + struct cfl_kvpair *pair; + + count = 0; + cfl_list_foreach(head, &list->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (strcmp(pair->key, key) == 0) { + count++; + } + } + + return count; +} + +/* the typed OTLP metadata of a merged sample must describe the last point */ +static void test_check_last_point_metadata(struct cmt_metric *sample, uint64_t start_time) +{ + struct cfl_variant *value; + + TEST_CHECK(sample->otlp_metadata != NULL); + if (sample->otlp_metadata == NULL) { + return; + } + + TEST_CHECK(test_kvlist_key_count(sample->otlp_metadata, "attributes") == 1); + TEST_CHECK(test_kvlist_key_count(sample->otlp_metadata, "start_time_unix_nano") == 1); + + value = cfl_kvlist_fetch(sample->otlp_metadata, "start_time_unix_nano"); + TEST_CHECK(value != NULL && + (value->type == CFL_VARIANT_UINT || value->type == CFL_VARIANT_INT)); + if (value != NULL && value->type == CFL_VARIANT_UINT) { + TEST_CHECK(value->data.as_uint64 == start_time); + } +} + +/* Repeated series inside one OTLP JSON request update one sample, last wins. */ +void test_opentelemetry_metrics_json_merges_repeated_series() +{ + int ret; + int result; + char *payload; + char *msgpack_buffer; + size_t msgpack_size; + flb_sds_t otlp_json; + struct cfl_list contexts; + struct cmt *context; + struct cmt_gauge *gauge; + struct cmt_counter *counter; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *sample; + + payload = + "{\"resourceMetrics\":[{\"resource\":{},\"scopeMetrics\":[{\"scope\":{\"name\":\"s\"}," + "\"metrics\":[" + "{\"name\":\"g\",\"gauge\":{\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\",\"asDouble\":1.0}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\",\"asDouble\":2.0}" + "]}}," + "{\"name\":\"u\",\"gauge\":{\"dataPoints\":[" + "{\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\",\"asDouble\":1.0}," + "{\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\",\"asDouble\":2.0}" + "]}}," + "{\"name\":\"c\",\"sum\":{\"aggregationTemporality\":2,\"isMonotonic\":true," + "\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\",\"asInt\":\"10\"}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\",\"asInt\":\"20\"}" + "]}}," + "{\"name\":\"h\",\"histogram\":{\"aggregationTemporality\":2,\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\"," + "\"count\":\"3\",\"sum\":3.0,\"bucketCounts\":[\"1\",\"2\"],\"explicitBounds\":[1.0]}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\"," + "\"count\":\"7\",\"sum\":9.0,\"bucketCounts\":[\"3\",\"4\"],\"explicitBounds\":[1.0]}" + "]}}," + "{\"name\":\"sm\",\"summary\":{\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\"," + "\"count\":\"2\",\"sum\":4.0," + "\"quantileValues\":[{\"quantile\":0.5,\"value\":1.0}]}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\"," + "\"count\":\"5\",\"sum\":8.0," + "\"quantileValues\":[{\"quantile\":0.5,\"value\":6.0}]}" + "]}}," + "{\"name\":\"e\",\"exponentialHistogram\":{\"aggregationTemporality\":2," + "\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\"," + "\"count\":\"1\",\"sum\":1.0,\"scale\":0,\"zeroCount\":\"0\"," + "\"positive\":{\"offset\":0,\"bucketCounts\":[\"1\"]}}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\"," + "\"count\":\"4\",\"sum\":6.0,\"scale\":0,\"zeroCount\":\"1\"," + "\"positive\":{\"offset\":1,\"bucketCounts\":[\"1\",\"2\"]}}" + "]}}" + "]}]}]}"; + + cfl_list_init(&contexts); + + ret = flb_opentelemetry_metrics_json_to_cmt(&contexts, payload, strlen(payload)); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&contexts) == 1); + if (ret != 0 || cfl_list_size(&contexts) != 1) { + destroy_metrics_context_list(&contexts); + return; + } + + context = cfl_list_entry_first(&contexts, struct cmt, _head); + + /* gauge with labels */ + gauge = TEST_FIND_FAMILY(&context->gauges, struct cmt_gauge, "g"); + TEST_CHECK(gauge != NULL); + sample = gauge != NULL ? test_single_dynamic_sample(gauge->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(cmt_metric_get_value(sample) == 2.0); + test_check_last_point_metadata(sample, 2); + } + + /* gauge without labels: both points land on the static sample */ + gauge = TEST_FIND_FAMILY(&context->gauges, struct cmt_gauge, "u"); + TEST_CHECK(gauge != NULL); + if (gauge != NULL) { + TEST_CHECK(gauge->map->metric_static_set == CMT_TRUE); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 0); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 2.0); + test_check_last_point_metadata(&gauge->map->metric, 2); + } + + /* sum */ + counter = TEST_FIND_FAMILY(&context->counters, struct cmt_counter, "c"); + TEST_CHECK(counter != NULL); + sample = counter != NULL ? test_single_dynamic_sample(counter->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(cmt_metric_get_value(sample) == 20.0); + test_check_last_point_metadata(sample, 2); + } + + /* histogram */ + histogram = TEST_FIND_FAMILY(&context->histograms, struct cmt_histogram, "h"); + TEST_CHECK(histogram != NULL); + sample = histogram != NULL ? test_single_dynamic_sample(histogram->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(cmt_metric_hist_get_count_value(sample) == 7); + TEST_CHECK(cmt_metric_hist_get_sum_value(sample) == 9.0); + TEST_CHECK(cmt_metric_hist_get_bucket_count(sample, 0) == 3); + TEST_CHECK(cmt_metric_hist_get_bucket_count(sample, 1) == 4); + test_check_last_point_metadata(sample, 2); + } + + /* summary */ + summary = TEST_FIND_FAMILY(&context->summaries, struct cmt_summary, "sm"); + TEST_CHECK(summary != NULL); + sample = summary != NULL ? test_single_dynamic_sample(summary->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(cmt_summary_get_count_value(sample) == 5); + TEST_CHECK(cmt_summary_get_sum_value(sample) == 8.0); + TEST_CHECK(cmt_summary_quantile_get_value(sample, 0) == 6.0); + test_check_last_point_metadata(sample, 2); + } + + /* exponential histogram */ + exp_histogram = TEST_FIND_FAMILY(&context->exp_histograms, struct cmt_exp_histogram, "e"); + TEST_CHECK(exp_histogram != NULL); + sample = exp_histogram != NULL ? test_single_dynamic_sample(exp_histogram->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(sample->exp_hist_count == 4); + TEST_CHECK(sample->exp_hist_zero_count == 1); + TEST_CHECK(sample->exp_hist_positive_offset == 1); + TEST_CHECK(sample->exp_hist_positive_count == 2); + if (sample->exp_hist_positive_count == 2) { + TEST_CHECK(sample->exp_hist_positive_buckets[1] == 2); + } + test_check_last_point_metadata(sample, 2); + } + + /* the re-encoded payload carries a single data point per series */ + ret = cmt_encode_msgpack_create(context, &msgpack_buffer, &msgpack_size); + TEST_CHECK(ret == 0); + if (ret == 0) { + otlp_json = flb_opentelemetry_metrics_msgpack_to_otlp_json(msgpack_buffer, + msgpack_size, + &result); + TEST_CHECK(otlp_json != NULL); + if (otlp_json != NULL) { + TEST_CHECK(strstr(otlp_json, "\"asInt\":\"20\"") != NULL); + TEST_CHECK(strstr(otlp_json, "\"asInt\":\"10\"") == NULL); + TEST_CHECK(strstr(otlp_json, "\"startTimeUnixNano\":\"1\"") == NULL); + flb_sds_destroy(otlp_json); + } + flb_free(msgpack_buffer); + } + + destroy_metrics_context_list(&contexts); +} + +static size_t test_count_substring(const char *haystack, const char *needle) +{ + size_t count; + const char *cursor; + + count = 0; + cursor = haystack; + while ((cursor = strstr(cursor, needle)) != NULL) { + count++; + cursor += strlen(needle); + } + + return count; +} + +/* A list with many invalid attributes is reported once, healthy ones are kept. */ +void test_json_invalid_attributes_warn_once() +{ + int ret; + int type; + int saved_stderr; + int index; + char *buffer; + char log_buffer[4096]; + size_t size; + size_t offset; + size_t log_size; + FILE *capture; + flb_sds_t json; + msgpack_unpacked unpacked; + struct cfl_kvlist *attributes; + + json = flb_sds_create("["); + TEST_ASSERT(json != NULL); + for (index = 0; index < 50; index++) { + flb_sds_printf(&json, "{\"key\":\"b%d\",\"value\":{\"bytesValue\":\"!!!\"}},", index); + } + flb_sds_cat_safe(&json, "{\"key\":\"ok\",\"value\":{\"stringValue\":\"kept\"}}]", 46); + + ret = flb_pack_json(json, flb_sds_len(json), &buffer, &size, &type, NULL); + TEST_ASSERT(ret == 0); + + offset = 0; + msgpack_unpacked_init(&unpacked); + TEST_ASSERT(msgpack_unpack_next(&unpacked, buffer, size, &offset)); + + attributes = cfl_kvlist_create(); + TEST_ASSERT(attributes != NULL); + + capture = tmpfile(); + TEST_ASSERT(capture != NULL); + + fflush(stderr); + saved_stderr = dup(fileno(stderr)); + TEST_ASSERT(saved_stderr >= 0); + dup2(fileno(capture), fileno(stderr)); + + ret = flb_otel_utils_clone_kvlist_from_otlp_json_array(attributes, &unpacked.data); + + fflush(stderr); + dup2(saved_stderr, fileno(stderr)); + close(saved_stderr); + + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_kvlist_count(attributes) == 1); + TEST_CHECK(cfl_kvlist_fetch(attributes, "ok") != NULL); + + rewind(capture); + log_size = fread(log_buffer, 1, sizeof(log_buffer) - 1, capture); + log_buffer[log_size] = '\0'; + fclose(capture); + + TEST_CHECK(test_count_substring(log_buffer, "invalid OTLP JSON attribute") == 1); + TEST_CHECK(strstr(log_buffer, "ignoring 50 invalid") != NULL); + TEST_MSG("captured log: %s", log_buffer); + + cfl_kvlist_destroy(attributes); + msgpack_unpacked_destroy(&unpacked); + flb_free(buffer); + flb_sds_destroy(json); +} + +static int test_logs_json_to_otlp_json(char *payload, flb_sds_t *out) +{ + int ret; + int result; + struct flb_log_event_encoder encoder; + struct flb_opentelemetry_otlp_logs_options options; + + *out = NULL; + + ret = flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + result = 0; + ret = flb_opentelemetry_logs_json_to_msgpack(&encoder, payload, strlen(payload), + "log", &result); + if (ret != 0 || result != 0) { + flb_log_event_encoder_destroy(&encoder); + return -1; + } + + memset(&options, 0, sizeof(options)); + options.logs_require_otel_metadata = FLB_TRUE; + options.logs_body_key = "log"; + + *out = flb_opentelemetry_logs_to_otlp_json(encoder.output_buffer, encoder.output_length, + &options, &result); + flb_log_event_encoder_destroy(&encoder); + + return *out != NULL ? 0 : -1; +} + +/* OTLP JSON bytesValue is base64 decoded and round trips as bytesValue. */ +void test_opentelemetry_logs_json_bytes_value() +{ + int ret; + char *payload; + flb_sds_t actual; + + payload = + "{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"rb\"," + "\"value\":{\"bytesValue\":\"AAE=\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"s\"}," + "\"logRecords\":[{\"timeUnixNano\":\"1700000001000000000\"," + "\"body\":{\"bytesValue\":\"aGVsbG8=\"}," + "\"attributes\":[{\"key\":\"padded\",\"value\":{\"bytesValue\":\"AAH/\"}}," + "{\"key\":\"unpadded\",\"value\":{\"bytesValue\":\"aGVsbG8\"}}," + "{\"key\":\"urlsafe\",\"value\":{\"bytesValue\":\"-_8\"}}," + "{\"key\":\"empty\",\"value\":{\"bytesValue\":\"\"}}]}]}]}]}"; + + ret = test_logs_json_to_otlp_json(payload, &actual); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(strstr(actual, "\"body\":{\"bytesValue\":\"aGVsbG8=\"}") != NULL); + TEST_CHECK(strstr(actual, "{\"key\":\"rb\",\"value\":{\"bytesValue\":\"AAE=\"}}") != NULL); + TEST_CHECK(strstr(actual, + "{\"key\":\"padded\",\"value\":{\"bytesValue\":\"AAH/\"}}") != NULL); + TEST_CHECK(strstr(actual, "{\"key\":\"unpadded\"," + "\"value\":{\"bytesValue\":\"aGVsbG8=\"}}") != NULL); + TEST_CHECK(strstr(actual, + "{\"key\":\"urlsafe\",\"value\":{\"bytesValue\":\"+/8=\"}}") != NULL); + TEST_CHECK(strstr(actual, "stringValue") == NULL); + TEST_MSG("actual: %s", actual); + flb_sds_destroy(actual); + } + + /* invalid base64 is rejected like other invalid values */ + payload = + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[{" + "\"timeUnixNano\":\"1700000001000000000\",\"body\":{\"bytesValue\":\"!!!\"}}]}]}]}"; + ret = test_logs_json_to_otlp_json(payload, &actual); + TEST_CHECK(ret != 0); + if (actual != NULL) { + flb_sds_destroy(actual); + } + + payload = + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[{" + "\"timeUnixNano\":\"1700000001000000000\",\"body\":{\"stringValue\":\"x\"}," + "\"attributes\":[{\"key\":\"b\",\"value\":{\"bytesValue\":\"A\"}}]}]}]}]}"; + ret = test_logs_json_to_otlp_json(payload, &actual); + TEST_CHECK(ret != 0); + if (actual != NULL) { + flb_sds_destroy(actual); + } +} + /* Test list */ +static void test_json_anyvalue_signed_bytes_and_invalid_attributes(void) +{ + const char *json = "[{\"key\":\"negative\",\"value\":{\"intValue\":\"-9223372036854775808\"}}," + "{\"key\":\"positive\",\"value\":{\"intValue\":\"9223372036854775807\"}}," + "{\"key\":\"bytes\",\"value\":{\"bytesValue\":\"AAH/\"}}," + "{\"key\":\"overflow\",\"value\":{\"intValue\":\"18446744073709551615\"}}," + "{\"key\":\"invalid\",\"value\":{\"boolValue\":\"wrong\"}}," + "{\"key\":\"healthy\",\"value\":{\"stringValue\":\"kept\"}}]"; + char *buffer; + size_t size; + size_t offset = 0; + int type; + msgpack_unpacked unpacked; + struct cfl_kvlist *attributes; + struct cfl_variant *value; + const unsigned char bytes[] = {0, 1, 255}; + + TEST_ASSERT(flb_pack_json(json, strlen(json), &buffer, &size, &type, NULL) == 0); + msgpack_unpacked_init(&unpacked); + TEST_ASSERT(msgpack_unpack_next(&unpacked, buffer, size, &offset)); + attributes = cfl_kvlist_create(); + TEST_ASSERT(attributes != NULL); + TEST_CHECK(flb_otel_utils_clone_kvlist_from_otlp_json_array(attributes, &unpacked.data) == 0); + value = cfl_kvlist_fetch(attributes, "negative"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_INT); + if (value != NULL && value->type == CFL_VARIANT_INT) { + TEST_CHECK(value->data.as_int64 == INT64_MIN); + } + value = cfl_kvlist_fetch(attributes, "positive"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_INT); + if (value != NULL && value->type == CFL_VARIANT_INT) { + TEST_CHECK(value->data.as_int64 == INT64_MAX); + } + value = cfl_kvlist_fetch(attributes, "bytes"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_BYTES); + if (value != NULL && value->type == CFL_VARIANT_BYTES) { + TEST_CHECK(cfl_sds_len(value->data.as_bytes) == sizeof(bytes)); + TEST_CHECK(memcmp(value->data.as_bytes, bytes, sizeof(bytes)) == 0); + } + TEST_CHECK(cfl_kvlist_fetch(attributes, "overflow") == NULL); + TEST_CHECK(cfl_kvlist_fetch(attributes, "invalid") == NULL); + value = cfl_kvlist_fetch(attributes, "healthy"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_STRING); + TEST_CHECK(cfl_kvlist_count(attributes) == 4); + cfl_kvlist_destroy(attributes); + msgpack_unpacked_destroy(&unpacked); + flb_free(buffer); +} + TEST_LIST = { + {"json_anyvalue_signed_bytes_invalid_attributes", test_json_anyvalue_signed_bytes_and_invalid_attributes}, { "hex_to_id", test_hex_to_id }, { "hex_to_id_error_cases", test_hex_to_id_error_cases }, { "convert_string_number_to_u64", test_convert_string_number_to_u64 }, @@ -3573,5 +4044,9 @@ TEST_LIST = { test_opentelemetry_metrics_otlp_proto_batches_empty_context }, { "opentelemetry_metrics_cutoff", test_opentelemetry_metrics_cutoff }, + { "opentelemetry_metrics_json_merges_repeated_series", + test_opentelemetry_metrics_json_merges_repeated_series }, + { "json_invalid_attributes_warn_once", test_json_invalid_attributes_warn_once }, + { "opentelemetry_logs_json_bytes_value", test_opentelemetry_logs_json_bytes_value }, { 0 } }; From dd6b22f9a135e0dc81eb43ab3f00e0bc81619842 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 17/61] tests: internal: fts: add processor tests Signed-off-by: Eduardo Silva --- tests/internal/CMakeLists.txt | 4 + tests/internal/fts.c | 1232 +++++++++++++++++++++++++++++++++ 2 files changed, 1236 insertions(+) create mode 100644 tests/internal/fts.c diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 93d079f7b2b..6ef8190c122 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -216,6 +216,10 @@ if(FLB_IN_OPENTELEMETRY) list(APPEND UNIT_TESTS_FILES opentelemetry_protobuf.c) endif() +if(FLB_LUAJIT AND FLB_YYJSON AND FLB_PROCESSOR_FTS) + list(APPEND UNIT_TESTS_FILES fts.c) +endif() + set(UNIT_TESTS_DATA data/fips/disabled.cnf data/tls/certificate.pem diff --git a/tests/internal/fts.c b/tests/internal/fts.c new file mode 100644 index 00000000000..013f0d951bf --- /dev/null +++ b/tests/internal/fts.c @@ -0,0 +1,1232 @@ +/* Fluent Bit: native log-policy regression tests, Apache-2.0. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "flb_tests_internal.h" +#include +#include +#include "../../plugins/processor_fts/fts.h" + +static msgpack_object *field(msgpack_object *map, const char *key) +{ + size_t i; + + if (!map || map->type != MSGPACK_OBJECT_MAP) { + return NULL; + } + for (i = 0; i < map->via.map.size; i++) { + if (map->via.map.ptr[i].key.type == MSGPACK_OBJECT_STR && + map->via.map.ptr[i].key.via.str.size == strlen(key) && + memcmp(map->via.map.ptr[i].key.via.str.ptr, key, strlen(key)) == 0) { + return &map->via.map.ptr[i].val; + } + } + return NULL; +} + +static void run_policy(const char *code, int grouped, const char *error_mode, + int expected_records, int expected_groups, int conditional, + const char *limit_key, const char *limit) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + struct flb_time ts; + msgpack_object *value; + void *output = NULL; + size_t output_size = 0; + int i; + int type; + int records = 0; + int groups = 0; + int ret; + int depth = 0; + const char *kind; + const char zero_id[16] = {0}; + msgpack_object empty_map = {.type = MSGPACK_OBJECT_MAP, .via.map = {0, NULL}}; + msgpack_object array_values[2] = { + {.type = MSGPACK_OBJECT_STR, .via.str = {1, "\xff"}}, + {.type = MSGPACK_OBJECT_STR, .via.str = {2, "ok"}} + }; + msgpack_object legacy_array = {.type = MSGPACK_OBJECT_ARRAY, .via.array = {2, array_values}}; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "lua_policy_test", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", code) == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "error_mode", error_mode) == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "instruction_limit", limit_key ? "1000000" : "10000") == 0); + if (limit_key) { + TEST_CHECK(flb_processor_unit_set_property_str(unit, limit_key, limit) == 0); + } + if (conditional) { + unit->condition = flb_condition_create(FLB_COND_OP_AND); + TEST_ASSERT(unit->condition != NULL); + TEST_CHECK(flb_condition_add_rule(unit->condition, "$kind", FLB_RULE_OP_EQ, + "selected", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); + } + TEST_ASSERT(flb_processor_init(processor) == 0); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (i = 0; i < 6; i++) { + if (grouped && i % 3 == 0) { + TEST_CHECK(flb_log_event_encoder_group_init(encoder) == 0); + if (grouped == 3 || grouped >= 5) { + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("batch"), FLB_LOG_EVENT_CSTRING_VALUE("original"), + FLB_LOG_EVENT_CSTRING_VALUE("schema"), FLB_LOG_EVENT_CSTRING_VALUE("otlp"), + FLB_LOG_EVENT_CSTRING_VALUE("resource_id"), FLB_LOG_EVENT_INT64_VALUE(INT64_MAX), + FLB_LOG_EVENT_CSTRING_VALUE("scope_id"), FLB_LOG_EVENT_INT64_VALUE(INT64_MIN)); + } + else { + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("batch"), FLB_LOG_EVENT_CSTRING_VALUE("original"), + FLB_LOG_EVENT_CSTRING_VALUE("schema"), FLB_LOG_EVENT_BOOLEAN_VALUE(FLB_FALSE)); + } + TEST_CHECK(ret == 0); + if (grouped == 5) { + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("duplicate"), FLB_LOG_EVENT_CSTRING_VALUE("one"), + FLB_LOG_EVENT_CSTRING_VALUE("duplicate"), FLB_LOG_EVENT_CSTRING_VALUE("two")); + TEST_CHECK(ret == 0); + } + if (grouped == 6) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("empty"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&empty_map)); + TEST_CHECK(ret == 0); + } + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("producer"), FLB_LOG_EVENT_CSTRING_VALUE("original")); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_group_header_end(encoder) == 0); + } + if (grouped == 10 && i == 1) { + TEST_CHECK(flb_log_event_encoder_group_init(encoder) == 0); + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("batch"), FLB_LOG_EVENT_CSTRING_VALUE("original"), + FLB_LOG_EVENT_CSTRING_VALUE("schema"), FLB_LOG_EVENT_BOOLEAN_VALUE(FLB_FALSE)); + TEST_CHECK(ret == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("producer"), FLB_LOG_EVENT_CSTRING_VALUE("original")); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_group_header_end(encoder) == 0); + } + kind = i % 2 ? "other" : "selected"; + TEST_CHECK(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(&ts, 1700000000 + i, i); + TEST_CHECK(flb_log_event_encoder_set_timestamp(encoder, &ts) == 0); + if (grouped == 3 || grouped >= 5) { + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("severity_number"), FLB_LOG_EVENT_INT64_VALUE(30), + FLB_LOG_EVENT_CSTRING_VALUE("trace_id"), FLB_LOG_EVENT_BINARY_VALUE(zero_id, sizeof(zero_id))); + TEST_CHECK(ret == 0); + } + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("index"), FLB_LOG_EVENT_INT64_VALUE(i), + FLB_LOG_EVENT_CSTRING_VALUE("kind"), + FLB_LOG_EVENT_CSTRING_VALUE(kind), + FLB_LOG_EVENT_CSTRING_VALUE("secret"), FLB_LOG_EVENT_CSTRING_VALUE("one"), + FLB_LOG_EVENT_CSTRING_VALUE("secret"), FLB_LOG_EVENT_CSTRING_VALUE("two")); + TEST_CHECK(ret == 0); + if (grouped == 3 || grouped >= 5) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("legacy_uint"), FLB_LOG_EVENT_UINT64_VALUE(UINT64_MAX)); + TEST_CHECK(ret == 0); + } + if (grouped == 9) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("legacy_array"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&legacy_array)); + TEST_CHECK(ret == 0); + } + if (grouped == 7) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("duplicate_utf8"), FLB_LOG_EVENT_CSTRING_VALUE("valid"), + FLB_LOG_EVENT_CSTRING_VALUE("duplicate_utf8"), FLB_LOG_EVENT_STRING_VALUE("\xff", 1)); + TEST_CHECK(ret == 0); + } + TEST_CHECK(flb_log_event_encoder_commit_record(encoder) == 0); + if (grouped && i % 3 == 2 && !(grouped == 2 && i == 5)) { + TEST_CHECK(flb_log_event_encoder_group_end(encoder) == 0); + if (grouped == 10 && i == 2) { + TEST_CHECK(flb_log_event_encoder_group_end(encoder) == 0); + } + } + } + ret = flb_processor_run(processor, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, &output, &output_size); + TEST_CHECK(ret == 0); + if (output_size > 0) { + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_CHECK(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type == FLB_LOG_EVENT_GROUP_START) { + TEST_CHECK(depth == 0); + depth++; + groups++; + value = field(event.metadata, "batch"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_STR && + value->via.str.size == 8 && memcmp(value->via.str.ptr, "original", 8) == 0); + if (grouped == 6) { + TEST_CHECK(field(event.body, "empty") == NULL); + } + if (grouped == 8) { + TEST_CHECK(field(event.body, "resource") == NULL); + TEST_CHECK(field(event.body, "scope") == NULL); + } + value = field(event.body, "producer"); + TEST_ASSERT(value != NULL); + if (expected_groups == 6) { + TEST_CHECK(value->type == MSGPACK_OBJECT_POSITIVE_INTEGER && value->via.u64 == records); + } + else { + TEST_CHECK(value->type == MSGPACK_OBJECT_STR && value->via.str.size == 8 && + memcmp(value->via.str.ptr, "original", 8) == 0); + } + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + TEST_CHECK(depth == 1); + depth--; + } + else { + TEST_CHECK(depth == !!grouped); + value = field(event.body, "index"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_POSITIVE_INTEGER); + if (expected_records == 6) { + TEST_CHECK(value->via.u64 == records); + } + if (conditional) { + TEST_CHECK(value->via.u64 == 2 * records + 1); + } + if (strcmp(error_mode, "keep") == 0) { + TEST_CHECK(field(event.body, "grow") == NULL); + TEST_CHECK(field(event.body, "verified") == NULL); + TEST_CHECK(event.body->via.map.size == (grouped == 3 ? 5 : 4)); + } + if (grouped == 3 || grouped >= 5) { + value = field(event.body, "legacy_uint"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->type == MSGPACK_OBJECT_POSITIVE_INTEGER && value->via.u64 == UINT64_MAX); + value = field(event.metadata, "severity_number"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->via.u64 == 30); + } + if (grouped == 8) { + value = field(event.metadata, "otlp"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_MAP); + TEST_CHECK(field(value, "attributes") == NULL); + TEST_CHECK(field(value, "event_name") != NULL); + } + if (grouped == 9) { + value = field(event.body, "legacy_array"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_ARRAY); + TEST_ASSERT(value->via.array.size == 3); + TEST_CHECK(value->via.array.ptr[0].type == MSGPACK_OBJECT_STR); + TEST_CHECK(value->via.array.ptr[0].via.str.size == 3); + TEST_CHECK(memcmp(value->via.array.ptr[0].via.str.ptr, "new", 3) == 0); + TEST_CHECK(value->via.array.ptr[1].via.str.size == 1); + TEST_CHECK((unsigned char) value->via.array.ptr[1].via.str.ptr[0] == 255); + } + records++; + if (field(event.body, "verified")) { + TEST_CHECK(field(event.body, "secret") == NULL); + TEST_CHECK(event.body->via.map.size == (grouped == 3 ? 4 : 3)); + } + } + } + TEST_CHECK(depth == 0); + flb_log_event_decoder_destroy(&decoder); + } + TEST_CHECK(records == expected_records); + TEST_CHECK(groups == expected_groups); + if (output && output != encoder->output_buffer) { + flb_free(output); + } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void generic_context_isolation(void) +{ + run_policy("function process(log) " + "assert(log.resource == nil and log.scope == nil and log.attributes == nil); " + "assert(log.context.attributes:get_string('producer') == 'original'); " + "assert(log.context.metadata:get_string('batch') == 'original'); " + "log.context.attributes:set('producer', log.body:get('index')); " + "log.body:remove('secret'); log.body:set('verified', true) end", + 1, "drop", 6, 6, 0, NULL, NULL); +} + +static void generic_read_does_not_split(void) +{ + run_policy("function process(log) assert(log.context.metadata:get_string('batch') == 'original') end", + 1, "drop", 6, 2, 0, NULL, NULL); +} + +static void plain_logs_do_not_gain_context(void) +{ + run_policy("function process(log) assert(log.context == nil and log.resource == nil); " + "log.body:remove('secret'); log.body:set('verified', true) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void conditional_drop_preserves_groups(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) return fb.DROP end", + 1, "drop", 3, 2, 1, NULL, NULL); +} + +static void drop_all_removes_empty_groups(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) return fb.DROP end", + 1, "drop", 0, 0, 0, NULL, NULL); +} + +static void duplicate_redaction_canonicalizes(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "fb.redact(log.body, {'secret'}); assert(log.body:len() == 3); " + "log.body:remove('secret'); log.body:set('verified', true) end", + 1, "drop", 6, 2, 0, NULL, NULL); +} + +static void errors_restore_body_and_context(void) +{ + run_policy("function process(log) log.body:remove('secret'); " + "log.context.attributes:set('producer','partial'); error('failure') end", + 1, "keep", 6, 2, 0, NULL, NULL); +} + +static void instruction_exhaustion_drops(void) +{ + run_policy("function process(log) while true do end end", + 1, "drop", 0, 0, 0, NULL, NULL); +} + +static void caught_budget_errors_terminate(void) +{ + run_policy("function process(log) while true do pcall(function() while true do end end) end end", + 0, "drop", 0, 0, 0, NULL, NULL); +} + +static void vm_memory_exhaustion_drops(void) +{ + run_policy("function process(log) local s=string.rep('x', 2000000); log.body:set('large',s) end", + 1, "drop", 0, 0, 0, "memory_limit", "1M"); +} + +static void native_memory_exhaustion_drops(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) local a=fb.array(); " + "for i=1,5000 do a:append(fb.int64(i)) end end", + 1, "drop", 0, 0, 0, "memory_limit", "1M"); +} + +static void strict_parsing_and_time(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "assert(fb.parse_json('18446744073709551615'):value() == '18446744073709551615'); " + "assert(fb.parse_json('-9223372036854775808'):value() == '-9223372036854775808'); " + "assert(not pcall(fb.parse_json, '{\"a\":1,\"a\":2}')); " + "assert(not pcall(fb.parse_json, '{} garbage')); " + "assert(not pcall(fb.parse_json, '18446744073709551616')); " + "assert(fb.parse_time('1970-01-01T01:00:00+01:00').sec == 0); " + "assert(fb.parse_time('2024-02-29T00:00:00.000000001Z').nsec == 1); " + "assert(not pcall(fb.parse_time,'2100-02-29T00:00:00Z')) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void output_growth_fails_closed(void) +{ + run_policy("function process(log) log.body:set('grow', string.rep('x',300000)) end", + 1, "drop", 0, 0, 0, "output_limit", "1M"); +} + +static void output_growth_keep_restores_whole_chunk(void) +{ + run_policy("function process(log) log.body:set('grow', string.rep('x',300000)) end", + 1, "keep", 6, 2, 0, "output_limit", "1M"); +} + +static void initialization_failures(void) +{ + const char *codes[] = {"while true do end", "function !syntax", "error('init failure')", + "function other() end", "require('ffi')", "io.open('/tmp/x')", NULL}; + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + int i; + + for (i = 0; codes[i]; i++) { + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "init_failure", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", codes[i]) == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "instruction_limit", "10000") == 0); + TEST_CHECK(flb_processor_init(processor) != 0); + flb_processor_destroy(processor); + flb_config_exit(config); + } +} + +static void unclosed_group_preserves_records(void) +{ + run_policy("function process(log) end", 2, "drop", 6, 2, 0, NULL, NULL); +} + +static void exact_opaque_context_ids(void) +{ + run_policy("function process(log) " + "assert(log.context.metadata:get('resource_id'):value() == '9223372036854775807'); " + "assert(log.context.metadata:get('scope_id'):value() == '-9223372036854775808'); " + "assert(log.resource and log.scope); " + "log.body:remove('secret'); log.body:set('verified',true) end", + 3, "drop", 6, 2, 0, NULL, NULL); +} + +static void context_replacement_during_collection(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local replacement=fb.map(); " + "for _,key in ipairs(log.context.metadata:keys()) do " + "replacement:set(key,log.context.metadata:get(key)) end; " + "log.body:remove('secret'); log.context.metadata=replacement; collectgarbage() end", + 1, "keep", 6, 2, 0, NULL, NULL); + run_policy("local fb=require('fluentbit'); function process(log) " + "log.body:remove('secret'); log.context.attributes=fb.map(); collectgarbage() end", + 1, "keep", 6, 2, 0, NULL, NULL); +} + +static void deep_mutable_views(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) local node=log.body; " + "for i=1,60 do node=node:ensure_map('child') end; " + "node:set('value',fb.int64(1)); assert(node:get_number('value')==1); " + "log.body:remove('child'); log.body:remove('secret'); " + "log.body:set('verified',true) end", + 1, "drop", 6, 2, 0, NULL, NULL); +} + +static void garbage_collector_is_private(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "assert(log.body.__gc == nil and fb.null.__gc == nil) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void trusted_policy_jit_compiles(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_processor_instance *instance; + struct fts_runtime *runtime; + lua_State *l; + const char *probe = "return jit.status(), require('jit.util').traceinfo(1)"; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_jit_test", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "jit", "true") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "instruction_limit", "0") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", + "local sum=0; for j=1,30 do for i=1,100000 do sum=sum+i end end; " + "function process(log) end") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + instance = unit->ctx; + runtime = instance->context; + l = runtime->vm->state; + TEST_CHECK(lua_gethookmask(l) == 0); + TEST_ASSERT(luaL_loadbuffer(l, probe, strlen(probe), "jit_test") == 0); + lua_rawgeti(l, LUA_REGISTRYINDEX, runtime->helpers_env_ref); + lua_setfenv(l, -2); + TEST_ASSERT(lua_pcall(l, 0, 2, 0) == 0); + TEST_CHECK(lua_toboolean(l, -2)); + TEST_CHECK(lua_istable(l, -1)); + lua_pop(l, 2); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void native_histogram_buckets_are_preserved(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct cmt *input; + struct cmt *output; + struct cmt_histogram *histogram; + struct cmt_histogram_buckets *buckets; + void *result; + size_t result_size; + int edit; + + for (edit = 0; edit < 2; edit++) { + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_histogram_test", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_METRICS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", edit ? + "function process(metric) metric.name = 'renamed' end" : + "function process(metric) end") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + input = cmt_create(); + TEST_ASSERT(input != NULL); + buckets = cmt_histogram_buckets_create(2, 4.0, 7.0); + TEST_ASSERT(buckets != NULL); + histogram = cmt_histogram_create(input, "", "", "native", "native", buckets, 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_CHECK(cmt_histogram_observe(histogram, 100, 2.0, 0, NULL) == 0); + TEST_CHECK(cmt_histogram_observe(histogram, 100, 5.0, 0, NULL) == 0); + TEST_CHECK(cmt_histogram_observe(histogram, 100, 8.0, 0, NULL) == 0); + result = NULL; + result_size = 0; + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_METRICS, "native", 6, + input, 0, &result, &result_size) == 0); + output = result; + TEST_ASSERT(output != NULL); + TEST_ASSERT(cfl_list_size(&output->histograms) == 1); + histogram = cfl_list_entry_first(&output->histograms, struct cmt_histogram, _head); + TEST_CHECK(strcmp(histogram->opts.name, edit ? "renamed" : "native") == 0); + TEST_CHECK(histogram->map->metric.hist_buckets[0] == 1); + TEST_CHECK(histogram->map->metric.hist_buckets[1] == 2); + TEST_CHECK(histogram->map->metric.hist_buckets[2] == 3); + TEST_CHECK(cmt_metric_hist_get_count_value(&histogram->map->metric) == 3); + TEST_CHECK(cmt_metric_hist_get_sum_value(&histogram->map->metric) == 15.0); + if (output != input) { + cmt_destroy(output); + } + cmt_destroy(input); + flb_processor_destroy(processor); + flb_config_exit(config); + } +} + +static void native_metric_scalar_preservation(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct cmt *input; + struct cmt *output; + struct cmt_gauge *gauge; + struct cmt_gauge *sibling; + struct cfl_list *head; + int found; + void *result; + size_t result_size; + int mode; + + for (mode = 0; mode < 4; mode++) { + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_metric_test", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_METRICS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", + "function process(metric) metric.name = 'renamed' end") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "error_mode", mode == 2 ? "drop" : "keep") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + input = cmt_create(); + TEST_ASSERT(input != NULL); + gauge = cmt_gauge_create(input, "", "", "native", "native", 0, NULL); + TEST_ASSERT(gauge != NULL); + gauge->map->metric_static_set = CMT_TRUE; + if (mode == 0) { + cmt_metric_set_int64(&gauge->map->metric, 100, INT64_MIN); + } + else if (mode == 3) { + /* uint64 values that fit int64 still run the policy. */ + cmt_metric_set_uint64(&gauge->map->metric, 100, 5); + } + else { + cmt_metric_set_uint64(&gauge->map->metric, 100, UINT64_MAX); + sibling = cmt_gauge_create(input, "", "", "healthy", "healthy", 0, NULL); + TEST_ASSERT(sibling != NULL); + TEST_ASSERT(cmt_gauge_set(sibling, 100, 42, 0, NULL) == 0); + } + result = NULL; + result_size = 0; + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_METRICS, "native", 6, + input, 0, &result, &result_size) == 0); + output = result; + TEST_ASSERT(output != NULL); + if (mode == 0) { + gauge = cfl_list_entry_first(&output->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(strcmp(gauge->opts.name, "renamed") == 0); + TEST_CHECK(cmt_metric_get_value_type(&gauge->map->metric) == CMT_METRIC_VALUE_INT64); + TEST_CHECK(cmt_metric_get_int64_value(&gauge->map->metric) == INT64_MIN); + } + else if (mode == 3) { + TEST_CHECK(cfl_list_size(&output->gauges) == 1); + gauge = cfl_list_entry_first(&output->gauges, struct cmt_gauge, _head); + TEST_CHECK(strcmp(gauge->opts.name, "renamed") == 0); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 5); + } + else if (mode == 1) { + TEST_CHECK(cfl_list_size(&output->gauges) == 2); + found = 0; + cfl_list_foreach(head, &output->gauges) { + sibling = cfl_list_entry(head, struct cmt_gauge, _head); + if (strcmp(sibling->opts.name, "native") == 0) { + found++; + TEST_CHECK(cmt_metric_get_value_type(&sibling->map->metric) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK(cmt_metric_get_uint64_value(&sibling->map->metric) == UINT64_MAX); + } + } + TEST_CHECK(found == 1); + } + else { + TEST_CHECK(cfl_list_size(&output->gauges) == 1); + } + if (mode == 1) { + /* Kept families are copied: the producer still owns its context. */ + TEST_CHECK(output != input); + TEST_CHECK(cfl_list_size(&input->gauges) == 2); + } + if (mode == 1 || mode == 2) { + found = 0; + cfl_list_foreach(head, &output->gauges) { + sibling = cfl_list_entry(head, struct cmt_gauge, _head); + if (strcmp(sibling->opts.name, "renamed") == 0) { + TEST_CHECK(cmt_metric_get_value(&sibling->map->metric) == 42); + found++; + } + } + TEST_CHECK(found == 1); + } + if (output != input) { + cmt_destroy(output); + } + cmt_destroy(input); + flb_processor_destroy(processor); + flb_config_exit(config); + } +} + +static void untouched_otel_fields_are_preserved(void) +{ + run_policy("function process(log) end", 3, "drop", 6, 2, 0, NULL, NULL); + run_policy("function process(log) log.body:set('edited', true) end", 3, "drop", 6, 2, 0, NULL, NULL); +} + +static void schema_promotion_is_record_local(void) +{ + const char *policy = "function process(log) if log.body:get('index'):number() == 0 then " + "log.context.metadata:set('schema', 'otlp') end end"; + + run_policy(policy, 1, "keep", 6, 2, 0, NULL, NULL); + run_policy(policy, 1, "drop", 5, 2, 0, NULL, NULL); +} + +static void reserved_timestamps_are_record_errors(void) +{ + run_policy("function process(log) if log.body:get('index'):number() == 0 then " + "log.timestamp.sec = 4294967294; log.timestamp.nsec = 1 end end", + 0, "keep", 6, 0, 0, NULL, NULL); + run_policy("function process(log) if log.body:get('index'):number() == 0 then " + "log.timestamp.sec = 4294967295; log.timestamp.nsec = 0 end end", + 0, "drop", 5, 0, 0, NULL, NULL); +} + +static void small_record_batch_uses_wire_budget(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + struct flb_time timestamp; + void *output; + size_t output_size; + int index; + int count; + int ret; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_wire_budget", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", "function process(log) end") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + flb_time_set(×tamp, 1, 0); + for (index = 0; index < 75000; index++) { + TEST_CHECK(flb_log_event_encoder_begin_record(encoder) == 0); + TEST_CHECK(flb_log_event_encoder_set_timestamp(encoder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("value"), FLB_LOG_EVENT_INT64_VALUE(index % 10)); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_commit_record(encoder) == 0); + } + output = NULL; + output_size = 0; + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_LOGS, "wire", 4, + encoder->output_buffer, encoder->output_length, &output, &output_size) == 0); + TEST_ASSERT(output != NULL); + TEST_CHECK(output_size < 3 * 1024 * 1024); + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + count = 0; + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { count++; } + TEST_CHECK(count == 75000); + flb_log_event_decoder_destroy(&decoder); + if (output != encoder->output_buffer) { flb_free(output); } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void event_metatable_cannot_bypass_validation(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local context=log.context; local metadata=log.metadata; local n=0; " + "rawset(log,'context',nil); setmetatable(log,{__index=function(_,key) " + "if key=='context' then n=n+1; if n==6 then " + "local otlp=metadata:ensure_map('otlp'); " + "otlp:set('severity_number',fb.int64(999)); otlp:set('trace_id',fb.bytes('abc')); " + "end; return context end end}) end", + 3, "drop", 0, 0, 0, NULL, NULL); +} + +static void replaced_and_redacted_values_do_not_accumulate_charges(void) +{ + run_policy("local fb=require('fluentbit'); local shared=fb.map(); " + "function process(log) for i=1,2000 do " + "shared:set('secret',true); fb.redact(shared,{'secret'}); " + "shared:remove('secret') end; " + "log.body:remove('secret'); log.body:set('verified',true) end", + 0, "drop", 6, 0, 0, "memory_limit", "1M"); +} + +static int exhaust_or_recover_memory(lua_State *l) +{ + static char payload[300000]; + char small[1024]; + size_t index; + + if (lua_touserdata(l, 1) == NULL) { + lua_gc(l, LUA_GCSTOP, 0); + lua_newtable(l); + for (index = 0; index < 10000; index++) { + memset(small, 'x', sizeof(small)); + snprintf(small, 32, "%zu", index); + lua_pushlstring(l, small, sizeof(small)); + lua_rawseti(l, -2, index + 1); + } + return luaL_error(l, "expected memory exhaustion"); + } + memset(payload, 'x', sizeof(payload)); + lua_pushlstring(l, payload, sizeof(payload)); + return 1; +} + +static void memory_exhaustion_does_not_poison_later_callbacks(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_processor_instance *instance; + struct fts_runtime *runtime; + lua_State *l; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "memory_recovery", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "code", "function process(log) end") == 0); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "memory_limit", "1M") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + instance = unit->ctx; + runtime = instance->context; + l = runtime->vm->state; + luaL_unref(l, LUA_REGISTRYINDEX, runtime->callback_ref); + lua_pushcfunction(l, exhaust_or_recover_memory); + runtime->callback_ref = luaL_ref(l, LUA_REGISTRYINDEX); + TEST_CHECK(fts_runtime_call(runtime, NULL, 1) == LUA_ERRMEM); + lua_settop(l, 0); + TEST_CHECK(fts_runtime_call(runtime, runtime, 1) == 0); + TEST_CHECK(lua_objlen(l, -1) == 300000); + lua_settop(l, 0); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void policies_cannot_stop_automatic_gc(void) +{ + run_policy("function process(log) " + "assert(not pcall(collectgarbage,'stop')); " + "log.body:remove('secret'); log.body:set('verified',true) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void duplicate_context_metadata_does_not_split_groups(void) +{ + run_policy("function process(log) end", 5, "drop", 6, 2, 0, NULL, NULL); +} + +static void removing_empty_context_field_is_preserved(void) +{ + run_policy("function process(log) log.context.attributes:remove('empty') end", + 6, "drop", 6, 2, 0, NULL, NULL); +} + +static void inherited_duplicate_invalid_utf8_is_preserved(void) +{ + run_policy("function process(log) log.body:set('edited',true) end", + 7, "drop", 6, 2, 0, NULL, NULL); +} + +static void json_source_limit_allows_large_scalar(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local text='\"'..string.rep('x',1048574)..'\"'; " + "assert(#fb.parse_json(text):value() == 1048574) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void documented_depth_is_enforced(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local text=string.rep('[',65)..string.rep(']',65); " + "assert(not pcall(fb.parse_json,text)) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void minimum_memory_initializes_all_signals(void) +{ + int signals[] = {FLB_PROCESSOR_LOGS, FLB_PROCESSOR_METRICS, FLB_PROCESSOR_TRACES}; + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + size_t index; + + for (index = 0; index < sizeof(signals) / sizeof(signals[0]); index++) { + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "minimum_memory", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, signals[index], "fts"); + TEST_ASSERT(unit != NULL); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "code", "function process(item) end") == 0); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "memory_limit", "262144") == 0); + TEST_CHECK(flb_processor_init(processor) == 0); + flb_processor_destroy(processor); + flb_config_exit(config); + } +} + +static void print_is_not_a_host_io_escape(void) +{ + run_policy("function process(log) assert(print==nil) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void json_duplicate_detection_is_byte_exact(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "assert(fb.parse_json([[{\"a\\u0000x\":1,\"a\\u0000y\":2,\"\":true}]]):len()==3); " + "assert(not pcall(fb.parse_json,[[{\"a\\u0000x\":1,\"a\\u0000x\":2}]])) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void log_edits_do_not_export_empty_read_placeholders(void) +{ + run_policy("function process(log) log.metadata:ensure_map('otlp'):set('event_name','changed') end", + 8, "drop", 6, 2, 0, NULL, NULL); +} + +static void mixed_integer_addition_preserves_exact_values(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local v=fb._fts_add(fb.int64(9007199254740991),fb.uint64(2)); " + "assert(v:type()=='int64' and v:value()=='9007199254740993'); " + "v=fb._fts_add(fb.int64('-9223372036854775808'),fb.uint64('18446744073709551615')); " + "assert(v:type()=='int64' and v:value()=='9223372036854775807'); " + "v=fb._fts_add(fb.int64('9223372036854775807'),fb.uint64(1)); " + "assert(v:type()=='uint64' and v:value()=='9223372036854775808'); " + "assert(not pcall(fb._fts_add,fb.int64(1),fb.uint64('18446744073709551615'))) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void invalid_mutations_do_not_accumulate_charges(void) +{ + run_policy("local fb=require('fluentbit'); local shared=fb.map(); " + "function process(log) for i=1,5000 do " + "assert(not pcall(function() shared:set('bad',{}) end)) end; " + "log.body:remove('secret'); log.body:set('verified',true) end", + 0, "drop", 6, 0, 0, "memory_limit", "1M"); +} + +static void native_nan_equality_preserves_unchanged_values(void) +{ + struct cfl_variant *left; + struct cfl_variant *right; + + left = cfl_variant_create_from_double(NAN); + TEST_ASSERT(left != NULL); + right = fts_copy_value(left, 0); + TEST_ASSERT(right != NULL); + TEST_CHECK(fts_values_equal(left, right, 0)); + cfl_variant_destroy(left); + cfl_variant_destroy(right); + left = cfl_variant_create_from_double(-0.0); + right = cfl_variant_create_from_double(0.0); + TEST_ASSERT(left != NULL && right != NULL); + TEST_CHECK(!fts_values_equal(left, right, 0)); + cfl_variant_destroy(left); + cfl_variant_destroy(right); +} + +static void array_insert_preserves_inherited_invalid_value(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local original=log.body:get('legacy_array'); local result=fb.array(); " + "result:append('new'); for i=1,original:len() do result:append(original:get(i)) end; " + "log.body:set('legacy_array',result) end", + 9, "drop", 6, 2, 0, NULL, NULL); +} + +static void nested_and_stray_group_markers_preserve_records(void) +{ + run_policy("function process(log) end", 10, "drop", 6, 3, 0, NULL, NULL); +} + +static void dropped_spans_remove_empty_resource_groups(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct ctrace *input; + struct ctrace_resource_span *resource; + struct ctrace_scope_span *scope; + void *output = NULL; + size_t output_size = 0; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "empty_traces", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_TRACES, "fts"); + TEST_ASSERT(unit != NULL); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "code", + "local fb=require('fluentbit'); function process(span) return fb.DROP end") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + input = ctr_create(NULL); + TEST_ASSERT(input != NULL); + resource = ctr_resource_span_create(input); + TEST_ASSERT(resource != NULL); + scope = ctr_scope_span_create(resource); + TEST_ASSERT(scope != NULL); + TEST_ASSERT(ctr_span_create(input, scope, "dropped", NULL) != NULL); + TEST_ASSERT(flb_processor_run(processor, 0, FLB_PROCESSOR_TRACES, "test", 4, + input, 0, &output, &output_size) == 0); + TEST_CHECK(output == input); + TEST_CHECK(cfl_list_is_empty(&input->span_list)); + TEST_CHECK(cfl_list_is_empty(&input->resource_spans)); + ctr_destroy(input); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void copied_values_preserve_binary_data_and_source_lifetime(void) +{ + struct cfl_kvlist *map; + struct cfl_variant *source; + struct cfl_variant *copy; + struct cfl_kvpair *pair; + char payload[] = {'a', '\0', '\xff'}; + + map = cfl_kvlist_create(); + TEST_ASSERT(map != NULL); + TEST_ASSERT(cfl_kvlist_insert_s(map, "string", 6, + cfl_variant_create_from_string_s(payload, sizeof(payload), CFL_TRUE)) == 0); + TEST_ASSERT(cfl_kvlist_insert_s(map, "bytes", 5, + cfl_variant_create_from_bytes(payload, sizeof(payload), CFL_TRUE)) == 0); + TEST_ASSERT(cfl_kvlist_insert_s(map, "empty", 5, + cfl_variant_create_from_string_s("", 0, CFL_TRUE)) == 0); + source = cfl_variant_create_from_kvlist(map); + TEST_ASSERT(source != NULL); + copy = fts_copy_value(source, 0); + TEST_ASSERT(copy != NULL); + memset(payload, 'x', sizeof(payload)); + cfl_variant_destroy(source); + pair = fts_find_pair(copy, "string", 6); + TEST_ASSERT(pair != NULL); + TEST_CHECK(pair->val->type == CFL_VARIANT_STRING); + TEST_CHECK(cfl_variant_size_get(pair->val) == 3); + TEST_CHECK(memcmp(pair->val->data.as_string, "a\0\xff", 3) == 0); + pair = fts_find_pair(copy, "bytes", 5); + TEST_ASSERT(pair != NULL); + TEST_CHECK(pair->val->type == CFL_VARIANT_BYTES); + TEST_CHECK(cfl_variant_size_get(pair->val) == 3); + TEST_CHECK(memcmp(pair->val->data.as_bytes, "a\0\xff", 3) == 0); + pair = fts_find_pair(copy, "empty", 5); + TEST_ASSERT(pair != NULL); + TEST_CHECK(cfl_variant_size_get(pair->val) == 0); + cfl_variant_destroy(copy); +} + +static void plain_log_fast_path_preserves_root_validation(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "log.metadata=fb.map() end", 0, "drop", 0, 0, 0, NULL, NULL); + run_policy("function process(log) log.context={} end", + 0, "drop", 0, 0, 0, NULL, NULL); + run_policy("function process(log) log.context.metadata:set('schema','otlp') end", + 1, "drop", 0, 0, 0, NULL, NULL); + run_policy("function process(log) log.metadata=nil end", + 0, "keep", 6, 0, 0, NULL, NULL); +} + +static void scalar_getters_preserve_types_and_live_paths(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local m=fb.map(); m:set('s','a'..string.char(0)..'b'); m:set('b',false); " + "m:set('i',fb.int64('-9007199254740991')); " + "m:set('u',fb.uint64('9007199254740991')); m:set('n',fb.null); " + "assert(m:get_string('s')=='a'..string.char(0)..'b'); " + "assert(m:get_bool('b')==false and m:get_number('i')==-9007199254740991); " + "assert(m:get_number('u')==9007199254740991); " + "assert(m:get_string('missing')==nil and m:get_bool('missing')==nil " + "and m:get_number('missing')==nil); " + "for _,k in ipairs({'b','i','n'}) do assert(not pcall(m.get_string,m,k)) end; " + "assert(not pcall(m.get_bool,m,'s') and not pcall(m.get_number,m,'n')); " + "m:set('u',fb.uint64('9007199254740992')); " + "assert(not pcall(m.get_number,m,'u')); " + "m:set('i',fb.int64('-9007199254740992')); " + "assert(not pcall(m.get_number,m,'i')); " + "m:set('bytes',fb.bytes('abc')); assert(not pcall(m.get_string,m,'bytes')); " + "local a=fb.parse_json('[1,\"s\",true,null]'); " + "assert(a:get_number(1)==1 and a:get_string(2)=='s' and a:get_bool(3)); " + "for _,k in ipairs({0,5,1.5,math.huge,'1'}) do " + "assert(not pcall(a.get_number,a,k)) end; " + "assert(not pcall(a.get_string,a,4) and not pcall(m.get_string,m,1)); " + "local child=m:ensure_map('child'); child:set('s','before'); " + "m:set('child',fb.parse_json('{\"s\":\"after\"}')); " + "assert(child:get_string('s')=='after'); m:remove('child'); " + "assert(not pcall(child.get_string,child,'s')); " + "log.body:remove('secret'); log.body:set('verified',true) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void native_result_arena_does_not_own_lua_snapshots(void) +{ + struct cfl_arena *arena; + struct cfl_variant *source; + struct cfl_variant *pooled; + struct cfl_variant *snapshot; + struct cfl_kvlist *map; + struct cfl_array *array; + char binary[] = {'a', '\0', (char) 0xff}; + + arena = cfl_arena_create(4096); + TEST_ASSERT(arena != NULL); + map = cfl_kvlist_create(); + array = cfl_array_create(2); + TEST_ASSERT(map != NULL && array != NULL); + TEST_ASSERT(cfl_kvlist_insert_bytes(map, "binary", binary, sizeof(binary), CFL_FALSE) == 0); + TEST_ASSERT(cfl_kvlist_insert_string(map, "duplicate", "first") == 0); + TEST_ASSERT(cfl_kvlist_insert_string(map, "duplicate", "second") == 0); + TEST_ASSERT(cfl_array_append_uint64(array, UINT64_MAX) == 0); + TEST_ASSERT(cfl_array_append_int64(array, INT64_MIN) == 0); + TEST_ASSERT(cfl_kvlist_insert_array(map, "array", array) == 0); + source = cfl_variant_create_from_kvlist(map); + TEST_ASSERT(source != NULL); + pooled = fts_copy_value_in(source, 0, arena); + TEST_ASSERT(pooled != NULL && pooled->arena == arena); + TEST_CHECK(fts_values_equal(source, pooled, 0)); + snapshot = fts_copy_value(pooled, 0); + TEST_ASSERT(snapshot != NULL && snapshot->arena == NULL); + cfl_variant_destroy(source); + TEST_CHECK(fts_values_equal(snapshot, pooled, 0)); + cfl_variant_destroy(pooled); + cfl_arena_destroy(arena); + TEST_CHECK(cfl_variant_size_get(fts_find_pair(snapshot, "binary", 6)->val) == sizeof(binary)); + TEST_CHECK(memcmp(fts_find_pair(snapshot, "binary", 6)->val->data.as_bytes, + binary, sizeof(binary)) == 0); + TEST_CHECK(snapshot->data.as_kvlist->arena == NULL); + cfl_variant_destroy(snapshot); +} + +/* A policy snapshot fits the minimum VM budget; a second commit tree does not. */ +static void edited_record_needs_one_policy_tree(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + struct flb_time timestamp; + msgpack_object *value; + char *payload; + void *output = NULL; + size_t output_size = 0; + int ret; + const char binary[] = {'a', '\0', (char) 0xff}; + msgpack_object empty_map = {.type = MSGPACK_OBJECT_MAP, .via.map = {0, NULL}}; + msgpack_object empty_array = {.type = MSGPACK_OBJECT_ARRAY, .via.array = {0, NULL}}; + + payload = flb_malloc(120000); + TEST_ASSERT(payload != NULL); + memset(payload, 'x', 120000); + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "single_snapshot", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "code", + "function process(log) log.body:set('edited',true) end") == 0); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "memory_limit", "262144") == 0); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "error_mode", "keep") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + TEST_ASSERT(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(×tamp, 1700000000, 0); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(encoder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("payload"), FLB_LOG_EVENT_STRING_VALUE(payload, 120000), + FLB_LOG_EVENT_CSTRING_VALUE("binary"), FLB_LOG_EVENT_BINARY_VALUE(binary, sizeof(binary)), + FLB_LOG_EVENT_CSTRING_VALUE("unsigned"), FLB_LOG_EVENT_UINT64_VALUE(UINT64_MAX), + FLB_LOG_EVENT_CSTRING_VALUE("signed"), FLB_LOG_EVENT_INT64_VALUE(INT64_MIN), + FLB_LOG_EVENT_CSTRING_VALUE("zero"), FLB_LOG_EVENT_DOUBLE_VALUE(-0.0), + FLB_LOG_EVENT_CSTRING_VALUE("null"), FLB_LOG_EVENT_NULL_VALUE(), + FLB_LOG_EVENT_CSTRING_VALUE("map"), FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&empty_map), + FLB_LOG_EVENT_CSTRING_VALUE("array"), FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&empty_array)); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(encoder) == 0); + TEST_ASSERT(flb_processor_run(processor, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, &output, &output_size) == 0); + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + TEST_ASSERT(flb_log_event_decoder_next(&decoder, &event) == 0); + value = field(event.body, "edited"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_BOOLEAN && value->via.boolean); + value = field(event.body, "payload"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_STR && value->via.str.size == 120000); + value = field(event.body, "binary"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_BIN); + TEST_CHECK(value->via.bin.size == sizeof(binary)); + TEST_CHECK(memcmp(value->via.bin.ptr, binary, sizeof(binary)) == 0); + value = field(event.body, "unsigned"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_POSITIVE_INTEGER && + value->via.u64 == UINT64_MAX); + value = field(event.body, "signed"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + value->via.i64 == INT64_MIN); + value = field(event.body, "zero"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_FLOAT64 && signbit(value->via.f64)); + value = field(event.body, "null"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_NIL); + value = field(event.body, "map"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_MAP && value->via.map.size == 0); + value = field(event.body, "array"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_ARRAY && value->via.array.size == 0); + flb_log_event_decoder_destroy(&decoder); + if (output != encoder->output_buffer) { + flb_free(output); + } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(processor); + flb_config_exit(config); + flb_free(payload); +} + +static void empty_metadata_is_materialized_independently(void) +{ + run_policy("local previous; function process(log) " + "if previous then previous:ensure_map('nested'):set('value','late') end; " + "assert(log.metadata:len()==0); " + "local set=log.metadata.set; assert(not pcall(set,{},'x',true)); " + "log.metadata:ensure_map('nested'):set('value','current'); " + "previous=log.metadata:copy() end", 0, "drop", 6, 0, 0, NULL, NULL); +} + +TEST_LIST = { + {"empty_metadata_cow_isolation", empty_metadata_is_materialized_independently}, + {"edited_record_one_policy_tree", edited_record_needs_one_policy_tree}, + {"native_result_arena_snapshot_lifetime", native_result_arena_does_not_own_lua_snapshots}, + {"scalar_getters_live_paths", scalar_getters_preserve_types_and_live_paths}, + {"copied_values_binary_source_lifetime", copied_values_preserve_binary_data_and_source_lifetime}, + {"plain_log_root_validation", plain_log_fast_path_preserves_root_validation}, + {"drop_spans_removes_empty_groups", dropped_spans_remove_empty_resource_groups}, + {"array_insert_inherited_invalid_value", array_insert_preserves_inherited_invalid_value}, + {"nested_stray_group_markers", nested_and_stray_group_markers_preserve_records}, + {"invalid_mutations_accounting", invalid_mutations_do_not_accumulate_charges}, + {"native_nan_equality", native_nan_equality_preserves_unchanged_values}, + {"mixed_integer_addition_exact", mixed_integer_addition_preserves_exact_values}, + {"log_edit_empty_placeholders", log_edits_do_not_export_empty_read_placeholders}, + {"minimum_memory_all_signals", minimum_memory_initializes_all_signals}, + {"print_host_io_unavailable", print_is_not_a_host_io_escape}, + {"json_duplicate_detection_byte_exact", json_duplicate_detection_is_byte_exact}, + {"duplicate_context_metadata_groups", duplicate_context_metadata_does_not_split_groups}, + {"remove_empty_context_field", removing_empty_context_field_is_preserved}, + {"inherited_duplicate_invalid_utf8", inherited_duplicate_invalid_utf8_is_preserved}, + {"json_source_limit_large_scalar", json_source_limit_allows_large_scalar}, + {"documented_depth_enforced", documented_depth_is_enforced}, + {"replaced_redacted_values_accounting", replaced_and_redacted_values_do_not_accumulate_charges}, + {"memory_exhaustion_callback_recovery", memory_exhaustion_does_not_poison_later_callbacks}, + {"policies_cannot_stop_automatic_gc", policies_cannot_stop_automatic_gc}, + {"event_metatable_cannot_bypass_validation", event_metatable_cannot_bypass_validation}, + {"untouched_otel_fields_are_preserved", untouched_otel_fields_are_preserved}, + {"schema_promotion_is_record_local", schema_promotion_is_record_local}, + {"reserved_timestamps_are_record_errors", reserved_timestamps_are_record_errors}, + {"small_record_batch_uses_wire_budget", small_record_batch_uses_wire_budget}, + {"trusted_policy_jit_compiles", trusted_policy_jit_compiles}, + {"native_histogram_buckets_are_preserved", native_histogram_buckets_are_preserved}, + {"native_metric_scalar_preservation", native_metric_scalar_preservation}, + {"generic_context_isolation", generic_context_isolation}, + {"generic_read_does_not_split", generic_read_does_not_split}, + {"plain_logs_do_not_gain_context", plain_logs_do_not_gain_context}, + {"conditional_drop_preserves_groups", conditional_drop_preserves_groups}, + {"drop_all_removes_empty_groups", drop_all_removes_empty_groups}, + {"duplicate_redaction_canonicalizes", duplicate_redaction_canonicalizes}, + {"errors_restore_body_and_context", errors_restore_body_and_context}, + {"instruction_exhaustion_drops", instruction_exhaustion_drops}, + {"caught_budget_errors_terminate", caught_budget_errors_terminate}, + {"vm_memory_exhaustion_drops", vm_memory_exhaustion_drops}, + {"native_memory_exhaustion_drops", native_memory_exhaustion_drops}, + {"strict_parsing_and_time", strict_parsing_and_time}, + {"output_growth_fails_closed", output_growth_fails_closed}, + {"output_growth_keep_restores_whole_chunk", output_growth_keep_restores_whole_chunk}, + {"initialization_failures", initialization_failures}, + {"unclosed_group_preserves_records", unclosed_group_preserves_records}, + {"exact_opaque_context_ids", exact_opaque_context_ids}, + {"context_replacement_during_collection", context_replacement_during_collection}, + {"deep_mutable_views", deep_mutable_views}, + {"garbage_collector_is_private", garbage_collector_is_private}, + {NULL, NULL} +}; From 156959c82be1a44e7398bcd0a7a123102116b7a8 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 00:44:01 -0600 Subject: [PATCH 18/61] tests: integration: add processor_fts scenarios Signed-off-by: Eduardo Silva --- .../scenarios/processor_fts/tests/test_fts.py | 346 ++++++++++ .../processor_fts/tests/test_fts_audits.py | 97 +++ .../tests/test_fts_native_interop.py | 288 +++++++++ .../processor_fts/tests/test_fts_pipeline.py | 300 +++++++++ .../processor_fts/tests/test_fts_policies.py | 598 ++++++++++++++++++ .../processor_fts/tests/test_fts_records.py | 524 +++++++++++++++ .../tests/test_fts_security_batches.py | 253 ++++++++ .../processor_fts/tests/test_fts_signals.py | 471 ++++++++++++++ .../tests/test_fts_signals_pipeline.py | 119 ++++ 9 files changed, 2996 insertions(+) create mode 100644 tests/integration/scenarios/processor_fts/tests/test_fts.py create mode 100644 tests/integration/scenarios/processor_fts/tests/test_fts_audits.py create mode 100644 tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py create mode 100644 tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py create mode 100644 tests/integration/scenarios/processor_fts/tests/test_fts_policies.py create mode 100644 tests/integration/scenarios/processor_fts/tests/test_fts_records.py create mode 100644 tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py create mode 100644 tests/integration/scenarios/processor_fts/tests/test_fts_signals.py create mode 100644 tests/integration/scenarios/processor_fts/tests/test_fts_signals_pipeline.py diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts.py b/tests/integration/scenarios/processor_fts/tests/test_fts.py new file mode 100644 index 00000000000..578303900e0 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts.py @@ -0,0 +1,346 @@ +"""FTS (Fluent Typed Scripting): ownership, exact values, and OTel groups.""" + +import json +import re +import requests +from pathlib import Path + +import pytest +import yaml + +from utils.fluent_bit_manager import FluentBitStartupError +from utils.test_service import FluentBitTestService + + +def make_service(tmp_path, code, *, grouped=False, error_mode="keep", + threaded=False, output_processor=False, second_processor=None, output_format="json_lines", runtime_options=None): + processor = {"name": "fts", "code": code, "call": "process_log", + "error_mode": error_mode} + processor.update(runtime_options or {}) + processors = ([{"name": "opentelemetry_envelope"}] if grouped else []) + [processor] + if second_processor: + processors.append(second_processor) + dummy = {"name": "dummy", "tag": "typed.logs", "rate": 4, "threaded": threaded, + "dummy": json.dumps({"message": "original", "secret": "sensitive", + "Secret": "case-sensitive", "optional": None, + "large": 18446744073709551615, + "nested": {"one": 1}, "values": [1, 2]})} + output = {"name": "stdout", "match": "*", "format": output_format} + (output if output_processor else dummy)["processors"] = {"logs": processors} + config = tmp_path / "typed.yaml" + config.write_text(yaml.safe_dump({ + "service": {"flush": 0.2, "grace": 1, "log_level": "info", "http_server": True, + "http_listen": "127.0.0.1", + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [dummy], "outputs": [output]}, + })) + return FluentBitTestService(str(config)) + + +def log_text(service): + return Path(service.flb.log_file).read_text(encoding="utf-8") + + +def records(service): + return [json.loads(line) for line in log_text(service).splitlines() if line.startswith("{")] + + +@pytest.mark.parametrize("output_processor", [False, True]) +def test_removed_libraries_cannot_be_recovered(tmp_path, output_processor): + code = r''' +local function check_packages() + for _, name in ipairs({"os", "io", "debug", "jit", "ffi", "package", "coroutine"}) do + assert(rawget(_G, name) == nil) + assert(getfenv == nil and setfenv == nil) + assert(not pcall(require, name)) + assert(not pcall(function() module(name) end)) + end + assert(module == nil and loadfile == nil and dofile == nil) + assert(not pcall(require, "fluentbit\0os")) + assert(type(require("fluentbit")) == "table") +end +check_packages() +function process_log(log) + check_packages() + log.body:set("message", "restricted") +end +''' + service = make_service(tmp_path, code, output_processor=output_processor) + try: + service.start() + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(record["message"] == "restricted" for record in output) + assert "edits rolled back" not in log_text(service) + finally: + service.stop() + + +TYPED_CODE = r''' +local fb = require("fluentbit") +local previous, retained +function process_log(event) + local body = event.body + assert(body:get_string("message") == "original") + assert(body:get_string("missing") == nil) + assert(not pcall(function() body:get_number("values") end)) + assert(body:get("large"):type() == "uint64") + assert(body:get("large"):value() == "18446744073709551615") + assert(not pcall(function() return body:get("large"):number() end)) + assert(body:get("optional"):type() == "null") + assert(body:get("missing") == nil) + body:set("copied", body:get("large")) + body:set("signed", fb.int64("-9223372036854775808")) + body:set("unsigned", fb.uint64("18446744073709551615")) + body:set("null", fb.null) + body:set("empty_map", fb.map()) + body:set("empty_array", fb.array()) + body:set("flag", true) + assert(body:get_bool("flag") == true) + body:set("fraction", 1.25) + assert(body:get_number("fraction") == 1.25) + local allowed = fb.map() + allowed:set("keep", true) + allowed:set("remove", true) + fb.keep_keys(allowed, {"keep"}) + assert(allowed:len() == 1 and allowed:get("remove") == nil) + fb.redact(allowed, {"keep"}, false) + assert(allowed:get("keep"):value() == false) + allowed:set("nul\0key", "nul\0value") + assert(allowed:get("nul\0key"):value() == "nul\0value") + assert(not pcall(function() fb.uint64(" 1") end)) + assert(not pcall(function() fb.uint64("1\0") end)) + assert(not pcall(function() fb.int64(9007199254740992) end)) + assert(not pcall(function() fb.attributes(event) end)) + body:set("utf8", "café") + body:get("nested"):set("one", fb.int64(7)) + local child = body:get("nested"):get("one") + body:remove("nested") + assert(not pcall(function() return child:value() end)) + body:set("nested", fb.map()) + body:get("nested"):set("one", fb.int64(8)) + assert(child:number() == 8) -- Views resolve paths after replacement. + body:get("values"):set(1, fb.map()) + body:get("values"):get(1):set("temporary", true) + body:get("values"):set(1, fb.int64(3)) + body:get("values"):append(fb.int64(4)) + assert(body:get("values"):len() == 3) + local binary = fb.bytes("a\0b") + assert(binary:type() == "bytes" and binary:value() == "a\0b") + body:set("binary", binary) + fb.redact(body, {"secret"}) + assert(body:get("Secret"):value() == "[REDACTED]") + if previous then + -- Copies are owned and cannot change already committed records. + previous:set("message", "late mutation") + assert(retained:value() == "18446744073709551615") + end + previous, retained = body:copy(), body:get("large"):copy() + body:set("message", "typed") + event.timestamp.sec, event.timestamp.nsec = 2147483648, 123456789 + return fb.KEEP +end +''' + + +@pytest.mark.parametrize("threaded,output_processor", [(False, False), (True, False), (False, True)]) +def test_typed_values_and_ownership(tmp_path, threaded, output_processor): + service = make_service(tmp_path, TYPED_CODE, threaded=threaded, output_processor=output_processor) + try: + service.start() + output = service.wait_for_condition(lambda: records(service) if len(records(service)) >= 3 else None, + timeout=30) + assert "callback failed" not in log_text(service) + for record in output: + assert record["message"] == "typed" + assert record["large"] == record["copied"] == record["unsigned"] == 18446744073709551615 + assert record["signed"] == -9223372036854775808 + assert record["null"] is None and record["optional"] is None + assert record["empty_map"] == {} and record["empty_array"] == [] + assert record["nested"] == {"one": 8} and record["values"] == [3, 2, 4] + assert record["secret"] == "[REDACTED]" and record["Secret"] == "[REDACTED]" + assert record["flag"] is True and record["fraction"] == 1.25 and record["utf8"] == "café" + finally: + service.stop() + + +@pytest.mark.parametrize("failure", [ + 'error("intentional")', + 'do return "invalid-result" end', + 'event.timestamp.nsec = 1000000000', + 'event.metadata = fb.array()', + 'event.body:set("bad", nil)', + 'fb.uint64("18446744073709551616")', + 'fb.int64("-9223372036854775809")', + 'fb.uint64(-1)', + 'event.body:get("values"):get(0)', +]) +def test_errors_rollback_record(tmp_path, failure): + code = f'''local fb = require("fluentbit") +function process_log(event) + event.body:set("message", "partial edit") + {failure} + return fb.KEEP +end''' + service = make_service(tmp_path, code) + try: + service.start() + service.wait_for_condition(lambda: "edits rolled back" in log_text(service), timeout=30) + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(record["message"] == "original" for record in output) + def errors_visible(): + response = requests.get(f"http://127.0.0.1:{service.flb.http_monitoring_port}/api/v2/metrics/prometheus", + timeout=5) + matches = re.findall(r'^fluentbit_processor_errors_total\{[^}]*processor="fts"[^}]*\} ([0-9.eE+-]+)', + response.text, re.MULTILINE) + return any(float(value) > 0 for value in matches) + service.wait_for_condition(errors_visible, timeout=30) + finally: + service.stop() + + +@pytest.mark.parametrize("explicit", [True, False]) +def test_record_drop(tmp_path, explicit): + code = '''local fb = require("fluentbit") +function process_log(event) + %s +end''' % ('return fb.DROP' if explicit else 'error("drop-on-error")') + service = make_service(tmp_path, code, grouped=True, error_mode="drop") + try: + service.start() + service.stop() + assert not records(service) + if not explicit: + assert "edits rolled back" in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("code", ["function !bad syntax", 'error("initializer-failed")', "function other() end"]) +def test_startup_failure_cleanup(tmp_path, code): + service = make_service(tmp_path, code) + try: + with pytest.raises(FluentBitStartupError): + service.start() + assert "FTS initialization failed" in log_text(service) + finally: + service.stop() + + +def test_context_aliases_survive_processor_chain(tmp_path): + code = '''local fb = require("fluentbit") +function process_log(log) + log.body:set("large", fb.int64("9223372036854775807")) + log.resource.attributes:set("service.name", "payments") + log.scope.name = "policy" + log.scope.version = "2" + log.scope.attributes:set("revision", fb.int64(3)) + log.attributes:set("redacted", true) + fb.redact(log.body, {"secret"}) +end''' + downstream = {"name": "fts", "code": '''function process(log) + assert(log.resource.attributes:get_string("service.name") == "payments") + assert(log.scope.name == "policy" and log.scope.version == "2") + assert(log.scope.attributes:get_number("revision") == 3) + assert(log.attributes:get_bool("redacted")) + log.body:set("verified", true) + end'''} + service = make_service(tmp_path, code, grouped=True, second_processor=downstream) + try: + service.start() + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(record["verified"] and record["secret"] == "[REDACTED]" for record in output) + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +def test_script_file_and_default_callback(tmp_path): + script = tmp_path / "policy.lua" + script.write_text('local fb=require("fluentbit"); function process(log) ' + 'log.body:set("message", "script-file"); log.body:set("count",fb.int64(1)) end') + service = make_service(tmp_path, "unused") + config = tmp_path / "typed.yaml" + document = yaml.safe_load(config.read_text()) + processor = document["pipeline"]["inputs"][0]["processors"]["logs"][0] + processor.pop("call"); processor.pop("code"); processor["script"] = str(script) + config.write_text(yaml.safe_dump(document)) + try: + service.start() + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(log["message"] == "script-file" and log["count"] == 1 for log in output) + finally: + service.stop() + + +@pytest.mark.parametrize("signal", ["profiles"]) +def test_unsupported_signals_fail_startup(tmp_path, signal): + service = make_service(tmp_path, "function process_log(log) end") + config = tmp_path / "typed.yaml" + document = yaml.safe_load(config.read_text()) + processors = document["pipeline"]["inputs"][0]["processors"] + processors[signal] = processors.pop("logs") + config.write_text(yaml.safe_dump(document)) + try: + with pytest.raises(FluentBitStartupError): + service.start() + assert "unknown processor 'profiles'" in log_text(service) + finally: + service.stop() + + +def test_unpublished_processor_name_is_not_registered(tmp_path): + service = make_service(tmp_path, "function process_log(log) end") + config = tmp_path / "typed.yaml" + document = yaml.safe_load(config.read_text()) + document["pipeline"]["inputs"][0]["processors"]["logs"][0]["name"] = "lua_typed" + config.write_text(yaml.safe_dump(document)) + try: + with pytest.raises(FluentBitStartupError): + service.start() + finally: + service.stop() + + +def test_record_edit_without_module_import(tmp_path): + service = make_service(tmp_path, """function process_log(log) + log.body:set("message", "no-import") + end""") + try: + service.start() + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(log["message"] == "no-import" for log in output) + finally: + service.stop() + + +@pytest.mark.parametrize("output_processor", [False, True]) +def test_trusted_jit_policy_runs(tmp_path, output_processor): + code = """local total=0 +for j=1,30 do for i=1,100000 do total=total+i end end +function process_log(log) + local sum=0; for i=1,50000 do sum=sum+i end + log.body:set('message', tostring(sum)) +end""" + service = make_service(tmp_path, code, output_processor=output_processor, + runtime_options={"jit": True, "instruction_limit": 0}) + try: + service.start() + rows = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(row["message"] == "1250025000" for row in rows) + assert "edits rolled back" not in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("options", [{"jit": True}, {"jit": True, "instruction_limit": 1000}, + {"jit": False, "instruction_limit": 0}]) +def test_jit_and_instruction_budget_cannot_be_combined(tmp_path, options): + service = make_service(tmp_path, "function process_log(log) end", runtime_options=options) + try: + with pytest.raises(FluentBitStartupError): + service.start() + expected = ("jit requires instruction_limit=0" if options.get("jit") else + "instruction_limit must be between 1000 and 2147483647 with jit disabled") + assert expected in log_text(service) + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_audits.py b/tests/integration/scenarios/processor_fts/tests/test_fts_audits.py new file mode 100644 index 00000000000..95dc5017806 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_audits.py @@ -0,0 +1,97 @@ +"""Regressions for preservation, read-only context access and timestamp writes.""" +import json +from pathlib import Path + +import pytest +import yaml + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port +from utils.test_service import FluentBitTestService +from test_fts_policies import PolicyRunner, attributes, scalar + + +@pytest.fixture(scope="module", params=["input", "output"]) +def audit_runner(request, tmp_path_factory): + port = find_available_port() + source = {"name": "opentelemetry", "port": port} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + code = '''function process(log) + local action = log.body:get_string("text") + if action == "edit" then log.body:set("edited", true) + elseif action == "read" then + assert(log.resource.attributes:len() >= 0) + assert(log.scope.attributes:len() >= 0) + elseif action == "marker" then + log.timestamp.sec = 4294967295; log.timestamp.nsec = 1 + end + end''' + (source if request.param == "input" else output)["processors"] = { + "logs": [{"name": "fts", "code": code, "error_mode": "drop"}]} + config = tmp_path_factory.mktemp("fts-audits") / "audit.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + manager = FluentBitManager(str(config)) + try: + manager.start() + yield PolicyRunner(manager, port) + finally: + manager.stop() + + +@pytest.mark.parametrize("action", ["noop", "edit", "read"]) +def test_untouched_otel_fields_and_context_survive(audit_runner, action): + request = f"audit-{action}" + payload = audit_runner.payload("unused", request) + source = payload["resourceLogs"][0] + source["scopeLogs"][0]["scope"].pop("attributes") + logs = source["scopeLogs"][0]["logRecords"] + for index, record in enumerate(logs): + record["traceId"] = "0" * 32 + record["spanId"] = "0" * 16 + record["severityNumber"] = 30 + if index < 2: + record["body"]["kvlistValue"]["values"][0]["value"]["stringValue"] = action + audit_runner.send(payload) + resources = audit_runner.wait_for_condition(lambda: audit_runner.collect(request)) + assert len(resources) == 1 + assert len(resources[0]["scopeLogs"]) == 1 + exported = resources[0]["scopeLogs"][0]["logRecords"] + assert len(exported) == 3 + for index, record in enumerate(exported): + assert record["traceId"] == "0" * 32 and record["spanId"] == "0" * 16 + assert record["severityNumber"] == 30 + assert ("edited" in scalar(record["body"])) == (action == "edit" and index < 2) + assert "edits rolled back" not in Path(audit_runner.manager.log_file).read_text() + + +def test_reserved_timestamp_drops_only_changed_record(audit_runner): + request = "audit-marker" + payload = audit_runner.payload("unused", request) + payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0]["body"]["kvlistValue"]["values"][0]["value"]["stringValue"] = "marker" + audit_runner.send(payload) + resources = audit_runner.wait_for_condition(lambda: audit_runner.collect(request)) + records = [record for resource in resources for scope in resource["scopeLogs"] + for record in scope["logRecords"]] + assert [scalar(record["body"])["text"] for record in records] == ["operationB", "control"] + + +def test_plain_metadata_does_not_gain_otlp_fields(tmp_path): + config = tmp_path / "plain.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": { + "inputs": [{"name": "dummy", "dummy": json.dumps({"message": "plain"}), + "metadata": json.dumps({"trace_flags": 1, "event_name": "ordinary.metadata"}), "rate": 4}], + "outputs": [{"name": "stdout", "match": "*", "format": "otlp_json"}]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + records = [record for batch in batches for resource in batch["resourceLogs"] + for scope in resource["scopeLogs"] for record in scope["logRecords"]] + assert records + assert all("flags" not in record and "eventName" not in record for record in records) + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py b/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py new file mode 100644 index 00000000000..c773175cfdc --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py @@ -0,0 +1,288 @@ +"""FTS extensions must preserve metric label processors and non-FTS exports.""" +import json +import os +import socket +from pathlib import Path + +import pytest +import requests +import yaml + +from utils.network import find_available_port +from utils.test_service import FluentBitTestService +from utils.fluent_bit_manager import FluentBitManager +from test_fts_signals import attr, metric, number_point +from google.protobuf.json_format import ParseDict +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from test_fts_policies import attributes + + +@pytest.mark.parametrize("fts", [False, True]) +@pytest.mark.parametrize("unlabeled", [False, True]) +def test_labels_and_static_labels_survive_otlp_export(tmp_path, fts, unlabeled): + source_port, destination_port = find_available_port(), find_available_port() + processors = ([{"name": "fts", "code": "function process(metric) end"}] if fts else []) + processors.append({"name": "labels", "insert": "environment production", "update": "region west", + "delete": "secret"}) + config = tmp_path / "labels.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": { + "inputs": [{"name": "opentelemetry", "port": source_port, "tag": "original", + "processors": {"metrics": processors}}, + {"name": "opentelemetry", "port": destination_port, "tag": "converted"}], + "outputs": [{"name": "opentelemetry", "match": "original", "host": "127.0.0.1", + "port": destination_port, "tls": "off", "add_label": "cluster test"}, + {"name": "stdout", "match": "converted", "format": "otlp_json"}]}})) + item = metric("gauge", "series") + points = [number_point(8), number_point(42)] + points[1]["startTimeUnixNano"] = "2147483649000000000" + points[1]["timeUnixNano"] = "2147483649123456789" + points[1]["flags"] = 3 + if unlabeled: + for point in points: + point["attributes"] = [] + item["gauge"]["dataPoints"] = points + payload = {"resourceMetrics": [{"scopeMetrics": [{"metrics": [item]}]}]} + service = FluentBitTestService(str(config)) + try: + service.start() + response = requests.post(f"http://127.0.0.1:{source_port}/v1/metrics", json=payload, timeout=10) + response.raise_for_status() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + exported = [p for b in batches for r in b.get("resourceMetrics", []) for s in r["scopeMetrics"] + for m in s["metrics"] for p in m["gauge"]["dataPoints"]] + # OTLP JSON input merges repeated series by labels, so the last point wins. + expected = [(42, "2147483649000000000", "2147483649123456789", 3)] + assert [(int(p["asInt"]), p["startTimeUnixNano"], p["timeUnixNano"], p["flags"]) + for p in exported] == expected + for point in exported: + values = attributes(point["attributes"]) + assert values["environment"] == "production" and values["cluster"] == "test" + assert "secret" not in values + if not unlabeled: + assert values["region"] == "west" + assert "cannot construct FTS" not in Path(service.flb.log_file).read_text() + finally: + service.stop() + + +@pytest.mark.parametrize("fts", [False, True]) +def test_metric_chunk_compatibility(tmp_path, fts): + if not os.environ.get("FTS_COMPAT_BINARY"): + # Always exercise an actual 5.1.0 chunk when that binary is unavailable. + test_5_1_0_histogram_fixture_survives_forward(tmp_path, fts) + return + forward_port, source_port = find_available_port(), find_available_port() + receiver_config = tmp_path / "receiver.yaml" + sender_config = tmp_path / "sender.yaml" + service = {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"} + receiver_config.write_text(yaml.safe_dump({"service": service, "pipeline": { + "inputs": [{"name": "forward", "listen": "127.0.0.1", "port": forward_port}], + "outputs": [{"name": "stdout", "match": "*"}]}})) + source = {"name": "opentelemetry", "port": source_port, "tag": "compat"} + if fts: + source["processors"] = {"metrics": [{"name": "fts", "code": "function process(metric) end"}]} + sender_config.write_text(yaml.safe_dump({"service": service, "pipeline": { + "inputs": [source], "outputs": [{"name": "forward", "match": "*", "host": "127.0.0.1", + "port": forward_port}]}})) + receiver = FluentBitManager(str(receiver_config), binary_path=os.environ["FTS_COMPAT_BINARY"]) + sender = FluentBitTestService(str(sender_config)) + items = [metric("gauge", "compat_gauge"), metric("sum", "compat_sum"), + metric("histogram", "compat_histogram")] + histogram = items[2]["histogram"]["dataPoints"][0] + histogram.update(bucketCounts=["5", "1", "3"], count="9", sum=12.0) + items[1]["sum"]["isMonotonic"] = False + value = {"stringValue": "nested"} + for _ in range(25): + value = {"kvlistValue": {"values": [{"key": "child", "value": value}]}} + items[0]["gauge"]["dataPoints"][0]["attributes"].append({"key": "deep", "value": value}) + payload = {"resourceMetrics": [{"scopeMetrics": [{"metrics": items}]}]} + try: + receiver.start() + sender.start() + message = ParseDict(payload, ExportMetricsServiceRequest()) + response = requests.post(f"http://127.0.0.1:{source_port}/v1/metrics", data=message.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, timeout=10) + response.raise_for_status() + text = sender.wait_for_condition(lambda: (text if all(name in text for name in ("compat_gauge", "compat_sum", "compat_histogram")) else None) + if (text := Path(receiver.log_file).read_text()) else None, timeout=30) + assert "cmt_decode_msgpack_create failed" not in text + assert " 8" in text or "=8" in text or "= 8" in text + histogram_line = next(line for line in text.splitlines() if "compat_histogram" in line) + # The internal wire format keeps cumulative buckets, as 5.1 expects. + assert "buckets = { 1=5, 2=6, +Inf=9 }, sum=12, count=9" in histogram_line + diagnostics = Path(sender.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + finally: + sender.stop() + receiver.stop() + + +@pytest.mark.parametrize("fts", [False, True]) +def test_5_1_0_histogram_fixture_survives_forward(tmp_path, fts): + port = find_available_port() + source = {"name": "forward", "listen": "127.0.0.1", "port": port} + if fts: + source["processors"] = {"metrics": [{"name": "fts", "code": "function process(metric) end"}]} + config = tmp_path / "legacy-histogram.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": [source], + "outputs": [{"name": "stdout", "match": "*", "format": "otlp_json"}]}})) + fixture = Path(__file__).resolve().parents[5] / "lib/cmetrics/tests/data/histogram-v5.1.0.msgpack" + assert fixture.is_file(), "the vendored 5.1.0 compatibility fixture is required" + data = fixture.read_bytes() + # PackedForward frame: [tag, binary chunk, {fluent_signal: metrics}]. + message = b"\x93\xa6legacy\xc6" + len(data).to_bytes(4, "big") + data + b"\x81\xadfluent_signal\x01" + service = FluentBitTestService(str(config)) + try: + service.start() + with socket.create_connection(("127.0.0.1", port), timeout=10) as connection: + connection.sendall(message) + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + rows = [m for b in batches for r in b.get("resourceMetrics", []) for s in r["scopeMetrics"] + for m in s["metrics"]] + assert len(rows) == 1 and rows[0]["name"] == "legacy_histogram" + point = rows[0]["histogram"]["dataPoints"][0] + assert [int(count) for count in point["bucketCounts"]] == [5, 1, 3] + assert int(point["count"]) == 9 + assert point["sum"] == 12.0 + diagnostics = Path(service.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("signal", ["metrics", "traces"]) +@pytest.mark.parametrize("export", ["stdout", "otlp"]) +def test_without_fts_json_anyvalue_preservation(tmp_path, signal, export): + from test_fts_signals import span + + source_port, destination_port = find_available_port(), find_available_port() + source = {"name": "opentelemetry", "port": source_port, "tag": "source"} + inputs = [source] + outputs = [{"name": "stdout", "match": "source", "format": "otlp_json"}] + if export == "otlp": + inputs.append({"name": "opentelemetry", "port": destination_port, "tag": "exported"}) + outputs = [{"name": "opentelemetry", "match": "source", "host": "127.0.0.1", + "port": destination_port}, + {"name": "stdout", "match": "exported", "format": "otlp_json"}] + config = tmp_path / "native-anyvalue.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": inputs, "outputs": outputs}})) + values = [attr("healthy", "kept"), + {"key": "negative", "value": {"intValue": "-9223372036854775808"}}, + {"key": "positive", "value": {"intValue": "9223372036854775807"}}, + {"key": "bytes", "value": {"bytesValue": "AAH/"}}, + {"key": "overflow", "value": {"intValue": "18446744073709551615"}}, + {"key": "invalid", "value": {"boolValue": "wrong"}}] + if signal == "metrics": + item = metric("gauge", "native-values") + item["gauge"]["dataPoints"][0]["attributes"] = values + resource_key, scope_key, item_key = "resourceMetrics", "scopeMetrics", "metrics" + else: + item = span("native-values") + item["attributes"] = values + resource_key, scope_key, item_key = "resourceSpans", "scopeSpans", "spans" + payload = {resource_key: [{scope_key: [{item_key: [item]}]}]} + service = FluentBitTestService(str(config)) + try: + service.start() + response = requests.post(f"http://127.0.0.1:{source_port}/v1/{signal}", json=payload, timeout=10) + response.raise_for_status() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + rows = [m for b in batches for r in b.get(resource_key, []) for s in r[scope_key] for m in s[item_key]] + assert len(rows) == 1 + entries = rows[0]["gauge"]["dataPoints"][0]["attributes"] if signal == "metrics" else rows[0]["attributes"] + fields = {entry["key"]: entry["value"] for entry in entries} + assert fields["healthy"] == {"stringValue": "kept"} + assert int(fields["negative"]["intValue"]) == -(2**63) + assert int(fields["positive"]["intValue"]) == 2**63-1 + assert fields["bytes"] == {"bytesValue": "AAH/"} + assert "overflow" not in fields and "invalid" not in fields + diagnostics = Path(service.flb.log_file).read_text() + assert "could not decode" not in diagnostics and "could not convert" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("export", ["stdout", "otlp"]) +def test_without_fts_unknown_signed_span_kind_keeps_siblings(tmp_path, export): + from test_fts_signals import span + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + import base64 + + source_port, destination_port = find_available_port(), find_available_port() + inputs = [{"name": "opentelemetry", "port": source_port, "tag": "source"}] + outputs = [{"name": "stdout", "match": "source", "format": "otlp_json"}] + if export == "otlp": + inputs.append({"name": "opentelemetry", "port": destination_port, "tag": "exported"}) + outputs = [{"name": "opentelemetry", "match": "source", "host": "127.0.0.1", "port": destination_port}, + {"name": "stdout", "match": "exported", "format": "otlp_json"}] + config = tmp_path / "signed-kind.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": inputs, "outputs": outputs}})) + items = [span("negative-kind"), span("healthy")] + items[0]["kind"] = -1 + for item in items: + for key in ("traceId", "spanId", "parentSpanId"): + item[key] = base64.b64encode(bytes.fromhex(item[key])).decode() + for link in item["links"]: + for key in ("traceId", "spanId"): + link[key] = base64.b64encode(bytes.fromhex(link[key])).decode() + request = ParseDict({"resourceSpans": [{"resource": {}, "scopeSpans": [{"scope": {}, "spans": items}]}]}, + ExportTraceServiceRequest()) + service = FluentBitTestService(str(config)) + try: + service.start() + response = requests.post(f"http://127.0.0.1:{source_port}/v1/traces", data=request.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, timeout=10) + response.raise_for_status() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + rows = [span for batch in batches for resource in batch.get("resourceSpans", []) + for scope in resource["scopeSpans"] for span in scope["spans"]] + assert [(row["name"], row["kind"]) for row in rows] == [("negative-kind", -1), ("healthy", 2)] + diagnostics = Path(service.flb.log_file).read_text() + assert "could not decode" not in diagnostics and "could not convert" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +@pytest.mark.parametrize("fts", [False, True]) +def test_log_trace_flags_preserve_the_low_eight_bits(tmp_path, transport, fts): + from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + + port = find_available_port() + source = {"name": "opentelemetry", "port": port} + if fts: + source["processors"] = {"logs": [{"name": "fts", "code": "function process(log) end"}]} + config = tmp_path / "flags.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": [source], + "outputs": [{"name": "stdout", "match": "*", "format": "otlp_json"}]}})) + payload = {"resourceLogs": [{"scopeLogs": [{"logRecords": [{"flags": 300, + "timeUnixNano": "2147483648123456789", "body": {"kvlistValue": {"values": [attr("marker", "flags")]}}}]}]}]} + service = FluentBitTestService(str(config)) + try: + service.start() + if transport == "protobuf": + message = ParseDict(payload, ExportLogsServiceRequest()) + response = requests.post(f"http://127.0.0.1:{port}/v1/logs", data=message.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, timeout=10) + else: + response = requests.post(f"http://127.0.0.1:{port}/v1/logs", json=payload, timeout=10) + response.raise_for_status() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + rows = [log for batch in batches for resource in batch.get("resourceLogs", []) + for scope in resource["scopeLogs"] for log in scope["logRecords"]] + assert len(rows) == 1 and rows[0]["flags"] == 44 + diagnostics = Path(service.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py b/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py new file mode 100644 index 00000000000..95afa656c69 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py @@ -0,0 +1,300 @@ +"""Real OTLP export, fan-out isolation, retry, and filesystem ingestion.""" +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +import requests +import yaml +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + +from utils.test_service import FluentBitTestService +from test_fts_policies import PolicyRunner, attributes, scalar + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("finish_native", [False, True]) +@pytest.mark.parametrize("native_neighbor", ["none", "before", "after"]) +def test_native_filter_chain_preserves_edits_and_groups(tmp_path, placement, finish_native, + native_neighbor): + script = tmp_path / "select.lua" + script.write_text('''function select_log(tag, timestamp, record) + if record.text ~= "operationA" then return -1, timestamp, record end + record.filter_ran = true + return 2, timestamp, record + end''') + processors = [{"name": "fts", "code": '''function process(log) + log.body:set("stage", "before") + log.resource.attributes:set("kept.text", log.body:get_string("text")) + end'''}, {"name": "lua", "script": str(script), "call": "select_log"}] + if native_neighbor != "none": + processors.insert(0 if native_neighbor == "before" else 1, + {"name": "content_modifier", "context": "body", "action": "insert", + "key": "native_ran", "value": "preserved"}) + if finish_native: + processors.append({"name": "fts", "code": '''function process(log) + assert(log.body:get_string("stage") == "before") + log.body:set("stage", "after") + end'''}) + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + (source if placement == "input" else output)["processors"] = {"logs": processors} + config = tmp_path / "mixed-chain.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + payload = PolicyRunner(None, 0).payload("unused", "mixed-chain") + logs = payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"] + logs.insert(2, json.loads(json.dumps(logs[0]))) + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/logs", + json=payload, timeout=15) + response.raise_for_status() + + def collect(): + rows = [] + for line in Path(service.flb.log_file).read_text().splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + batch = json.loads(line) + rows.extend((resource, log) for resource in batch["resourceLogs"] + for scope in resource["scopeLogs"] for log in scope["logRecords"]) + return rows if len(rows) >= 2 else None + + rows = service.wait_for_condition(collect, timeout=30) + assert len(rows) == 2 + for resource, log in rows: + body = scalar(log["body"]) + assert body["text"] == "operationA" + if native_neighbor != "none": + assert body["native_ran"] == "preserved" + assert body["filter_ran"] is True + assert body["stage"] == ("after" if finish_native else "before") + assert attributes(resource["resource"]["attributes"])["kept.text"] == "operationA" + assert all(scope["logRecords"] for scope in resource["scopeLogs"]) + diagnostics = Path(service.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + finally: + service.stop() + + +def test_group_end_markers_reach_native_processors(tmp_path): + """Native processors after FTS receive complete group end records.""" + processors = [{"name": "fts", "code": "function process(log) log.body:set('seen', true) end"}, + {"name": "sql", "query": "SELECT text, seen FROM STREAM;"}] + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", + "processors": {"logs": processors}} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + config = tmp_path / "fts-sql.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + payload = PolicyRunner(None, 0).payload("unused", "fts-sql") + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/logs", + json=payload, timeout=15) + response.raise_for_status() + + def collect(): + rows = [] + for line in Path(service.flb.log_file).read_text().splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + batch = json.loads(line) + rows.extend(log for resource in batch["resourceLogs"] + for scope in resource["scopeLogs"] for log in scope["logRecords"]) + return rows or None + + rows = service.wait_for_condition(collect, timeout=30) + assert rows and all(scalar(log["body"])["seen"] is True for log in rows) + finally: + service.stop() + +def protobuf_string_field(message, number): + # The suite's pinned OTel Python bindings predate event_name. Protobuf + # retains unknown fields, so verify field 12 on the exported wire itself. + data = message.SerializeToString() + position = 0 + result = [] + + def varint(): + nonlocal position + value, shift = 0, 0 + while True: + byte = data[position]; position += 1 + value |= (byte & 127) << shift + if byte < 128: return value + shift += 7 + assert shift < 70 + + while position < len(data): + key = varint(); wire = key & 7 + if wire == 0: varint() + elif wire == 1: position += 8 + elif wire == 5: position += 4 + else: + assert wire == 2 + length = varint() + if key >> 3 == number: + result.append(data[position:position + length].decode("utf-8")) + position += length + assert position <= len(data) + return result + + +@pytest.mark.parametrize("filesystem,replay", [(False, False), (True, False), (True, True)]) +def test_output_policy_isolated_across_routes_and_retries(tmp_path, filesystem, replay): + received, attempts = [], [] + blocked = {"value": replay} + + class Collector(BaseHTTPRequestHandler): + def do_POST(self): + message = ExportLogsServiceRequest.FromString(self.rfile.read(int(self.headers["Content-Length"]))) + attempts.append(message) + status = 503 if blocked["value"] or len(attempts) == 1 else 200 + if status == 200: + received.append(message) + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Collector) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + policy = '''function process(log) + local producer = log.body:get_string("text") + log.resource.attributes:set("service.name", producer) + log.scope.name = producer + log.body:set("export.policy", "sanitized") + log.body:remove("secret") + end''' + source = {"name":"opentelemetry", "port":"${FLUENT_BIT_TEST_LISTENER_PORT}"} + service_config = {"flush":0.2, "grace":1, "http_server":True, + "http_port":"${FLUENT_BIT_HTTP_MONITORING_PORT}", + "scheduler.base":1, "scheduler.cap":1} + if filesystem: + service_config["storage.path"] = str(tmp_path / "storage") + source["storage.type"] = "filesystem" + config = tmp_path / "pipeline.yaml" + config.write_text(yaml.safe_dump({"service":service_config, "pipeline":{ + "inputs":[source], "outputs":[ + {"name":"stdout", "match":"*", "format":"otlp_json"}, + {"name":"opentelemetry", "match":"*", "host":"127.0.0.1", "port":server.server_port, + "retry_limit":False, "processors":{"logs":[{"name":"fts", "code":policy}]}}, + ]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + payload = PolicyRunner(None, 0).payload("unused", "route-isolation") + payload["resourceLogs"][0]["resource"]["droppedAttributesCount"] = 7 + payload["resourceLogs"][0]["scopeLogs"][0]["scope"]["droppedAttributesCount"] = 5 + logs = payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"] + logs.insert(2, json.loads(json.dumps(logs[0]))) + for log in logs: + log["body"]["kvlistValue"]["values"].append( + {"key":"secret", "value":{"stringValue":"sensitive"}}) + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/logs",json=payload, timeout=15) + assert response.status_code in (200, 201) + if replay: + service.wait_for_condition(lambda: attempts, timeout=30) + service.stop() + original_log = Path(service.flb.log_file).read_text() + assert list((tmp_path / "storage").rglob("*.flb")), "filesystem chunk was not retained" + blocked["value"] = False + service = FluentBitTestService(str(config)) + service.start() + service.wait_for_condition(lambda: received, timeout=30) + service.stop() + if not replay: + original_log = Path(service.flb.log_file).read_text() + assert len(attempts) >= 2 + assert attempts[0].SerializeToString() == attempts[1].SerializeToString() + rows = [(resource, scope, log) for message in received for resource in message.resource_logs + for scope in resource.scope_logs for log in scope.log_records] + assert [dict((v.key, v.value.string_value) for v in log.body.kvlist_value.values)["text"] + for _, _, log in rows] == ["operationA", "operationB", "operationA", "control"] + for resource, scope, log in rows: + body = {v.key:v.value.string_value for v in log.body.kvlist_value.values} + assert body["export.policy"] == "sanitized" and "secret" not in body + assert {v.key:v.value.string_value for v in resource.resource.attributes}["service.name"] == body["text"] + assert scope.scope.name == body["text"] + assert resource.resource.dropped_attributes_count == 7 + assert scope.scope.dropped_attributes_count == 5 + assert log.dropped_attributes_count in (1, 2) + assert protobuf_string_field(log, 12) == ["application.event"] + assert log.time_unix_nano == 2147483648123456789 and log.flags in (1, 2) + raw_batches = [json.loads(line) for line in original_log.splitlines() + if line.startswith('{')] + raw_logs = [log for batch in raw_batches for resource in batch["resourceLogs"] + for scope in resource["scopeLogs"] for log in scope["logRecords"]] + assert len(raw_logs) == 4 + assert all(scalar(log["body"])["secret"] == "sensitive" for log in raw_logs) + assert all("export.policy" not in scalar(log["body"]) for log in raw_logs) + for batch in raw_batches: + for resource in batch["resourceLogs"]: + assert "service.name" not in attributes(resource["resource"]["attributes"]) + assert all(scope["scope"]["name"] == "scope" for scope in resource["scopeLogs"]) + finally: + service.stop() + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("rollback", [False, True]) +def test_pooled_batch_commit_rollback_and_retained_values(tmp_path, placement, rollback): + from test_fts_security_batches import BatchRunner + + code = ''' + local saved + function process(log) + if saved then + assert(saved:get_string("text") == "retained snapshot") + saved:set("text", "changed after commit") + end + log.body:set("text", "retained snapshot") + log.body:set("edited", true) + saved = log.body:copy() + if ROLLBACK and log.body:get_number("index") == 599 then + log.body:set("large", string.rep("x", 70000)) + end + end + '''.replace("ROLLBACK", "true" if rollback else "false") + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + (source if placement == "input" else output)["processors"] = {"logs": [ + {"name": "fts", "code": code, "error_mode": "keep", "output_limit": "65536"}]} + config = tmp_path / "pooled-batch.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + runner = BatchRunner(service.flb, service.flb_listener_port, "logs") + items = [{"timeUnixNano": "2147483648123456789", "body": {"kvlistValue": {"values": [ + {"key": "index", "value": {"intValue": str(index)}}, + {"key": "text", "value": {"stringValue": "original"}}]}}} for index in range(600)] + for _ in range(2): + _, rows = runner.send(items, "json") + assert len(rows) == 600 + bodies = [scalar(row["body"]) for row in rows] + assert [body["index"] for body in bodies] == list(range(600)) + assert all(body["text"] == ("original" if rollback else "retained snapshot") + for body in bodies) + assert all(("edited" not in body) if rollback else body["edited"] is True + for body in bodies) + diagnostics = runner.recent_log() + assert "policy failed" not in diagnostics + assert ("cannot construct policy output" in diagnostics) is rollback + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_policies.py b/tests/integration/scenarios/processor_fts/tests/test_fts_policies.py new file mode 100644 index 00000000000..8efb0342f24 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_policies.py @@ -0,0 +1,598 @@ +"""Log-policy matrix inspired by OTTL's log transform processor tests. + +These are independent end-to-end assertions, not an OTTL interpreter or a +claim of parity with the complete OTTL function catalog. Every policy runs +through both input and output processor placement and OTLP JSON export. +""" + +import copy +import json +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest +import requests +import yaml + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port + + +# (name, record policy, inherited-context policy, expected target, values for A/B) +POLICIES = [ + ("set", 'attrs:set("test", "pass")', "", "attrs", ["pass", "pass"]), + ("condition_true", 'if body:get_string("text") == "operationA" then attrs:set("test", "pass") end', + "", "attrs", ["pass", None]), + ("condition_false", 'if body:get_string("text") == "absent" then attrs:set("test", "pass") end', + "", "attrs", [None, None]), + ("resource_condition", 'if event.resource.attributes:get_string("host.name") == "localhost" ' + 'then attrs:set("test", "pass") end', "", "attrs", ["pass", "pass"]), + ("scope_condition", 'if event.scope.name == "scope" ' + 'then attrs:set("test", "pass") end', "", "attrs", ["pass", "pass"]), + ("scope_wrong_version", 'if event.scope.version == "2" ' + 'then attrs:set("test", "pass") end', "", "attrs", [None, None]), + ("severity_text", 'if attrs:get_string("http.path") == "/health" then otlp:set("severity_text", "OK") end', + "", "severityText", ["OK", "OK"]), + ("severity_number", 'if otlp:get_number("severity_number") == 9 then ' + 'otlp:set("severity_number", fb.int64(10)) end', "", "severityNumber", [10, 10]), + ("flags_condition", 'if fb.log_field(event, "trace_flags"):number() == 1 then attrs:set("test", "pass") end', + "", "attrs", ["pass", None]), + ("dropped_count_condition", 'if otlp:get_number("dropped_attributes_count") == 1 then ' + 'attrs:set("test", "pass") end', "", "attrs", ["pass", None]), + ("trace_id_condition", 'if otlp:get("trace_id"):value() == string.char(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16) ' + 'then attrs:set("test", "pass") end', "", "attrs", ["pass", "pass"]), + ("span_id_condition", 'if otlp:get("span_id"):value() == string.char(1,2,3,4,5,6,7,8) ' + 'then attrs:set("test", "pass") end', "", "attrs", ["pass", "pass"]), + ("pattern_condition", 'if body:get_string("text"):match("operation[AC]") then ' + 'attrs:set("test", "pass") end', "", "attrs", ["pass", None]), + ("replace_pattern", 'attrs:set("http.method", attrs:get_string("http.method"):gsub("g.t", "post"))', + "", "http.method", ["post", "post"]), + ("replace_all_values", 'for _, k in ipairs(attrs:keys()) do local v = attrs:get(k); ' + 'if v:type() == "string" then attrs:set(k, (v:value():gsub("get", "post"))) end end', + "", "http.method", ["post", "post"]), + ("rename_key", 'attrs:set("url", attrs:get("http.url")); attrs:remove("http.url")', + "", "rename", ["http://localhost/health", "http://localhost/health"]), + ("delete_key", 'if body:get_string("text") == "operationA" then attrs:remove("http.url") end', + "", "delete", [True, False]), + ("delete_matching_keys", 'for _, k in ipairs(attrs:keys()) do ' + 'if k:match("^http%..*t") then attrs:remove(k) end end', "", "delete_matching", [True, True]), + ("keep_keys", 'fb.keep_keys(attrs, {"http.method", "request.id"})', "", "keep", [True, True]), + ("concat", 'attrs:set("test", attrs:get_string("http.method") .. ": " .. attrs:get_string("http.url"))', + "", "attrs", ["get: http://localhost/health", "get: http://localhost/health"]), + ("split", 'local a = fb.array(); for s in attrs:get_string("flags"):gmatch("[^|]+") do ' + 'a:append(s) end; attrs:set("test", a)', "", "attrs", [["A", "B", "C"], ["C", "D"]]), + ("split_missing", 'local s = attrs:get_string("absent"); if s then ' + 'attrs:set("test", s:sub(1, 1)) end', "", "attrs", [None, None]), + ("substring", 'attrs:set("test", attrs:get_string("total.string"):sub(4, 6))', + "", "attrs", ["456", "678"]), + ("substring_missing", 'local s = attrs:get_string("absent"); if s then ' + 'attrs:set("test", s:sub(4, 6)) end', "", "attrs", [None, None]), + ("array_literal", 'local a = fb.array(); for _, s in ipairs({"A", "B", "C"}) do ' + 'a:append(s) end; attrs:set("test", a)', "", "attrs", [["A", "B", "C"], ["A", "B", "C"]]), + ("lower", 'attrs:set("test", body:get_string("text"):lower())', + "", "attrs", ["operationa", "operationb"]), + ("upper", 'attrs:set("test", body:get_string("text"):upper())', + "", "attrs", ["OPERATIONA", "OPERATIONB"]), + ("snake", 'attrs:set("test", (body:get_string("text"):gsub("(%l)(%u)", "%1_%2")):lower())', + "", "attrs", ["operation_a", "operation_b"]), + ("camel", 'local s = body:get_string("snake"); ' + 'attrs:set("test", (s:gsub("_(%l)", string.upper)))', + "", "attrs", ["operationA", "operationB"]), + ("merge_insert", 'local values = { ["http.method"] = "post", json_test = "pass" }; ' + 'for k, v in pairs(values) do if not attrs:get(k) then attrs:set(k, v) end end', + "", "merge_insert", ["pass", "pass"]), + ("merge_update", 'local values = { ["http.method"] = "post", absent = "ignored" }; ' + 'for k, v in pairs(values) do if attrs:get(k) then attrs:set(k, v) end end', + "", "http.method", ["post", "post"]), + ("merge_upsert", 'for k, v in pairs({ ["http.method"] = "post", new = "added" }) do ' + 'attrs:set(k, v) end', "", "merge_upsert", ["added", "added"]), + ("limit_zero", 'fb.keep_keys(attrs, {"request.id"})', "", "empty", [True, True]), + ("math_log", 'attrs:set("test", math.log(1))', "", "attrs", [0.0, 0.0]), + ("nested_body", 'body:get("metadata"):set("uid", "12345")', + "", "nested", ["12345", "12345"]), + ("record_cache", 'local cache = { text = body:get_string("text") }; attrs:set("test", cache.text)', + "", "attrs", ["operationA", "operationB"]), + ("resource_set", "", 'event.resource.attributes:set("test", "pass")', + "resource", "pass"), + ("resource_false", "", 'local a = event.resource.attributes; if a:get_string("host.name") == "wrong" ' + 'then a:set("test", "pass") end', "resource", None), + ("resource_schema", "", 'event.resource.schema_url = "https://schema.example/resource"', + "resource_schema", "https://schema.example/resource"), + ("scope_set", "", 'event.scope.attributes:set("test", "pass")', "scope", "pass"), + ("scope_schema", "", 'event.scope.schema_url = "https://schema.example/scope"', + "scope_schema", "https://schema.example/scope"), + ("scope_name", "", 'event.scope.name = "new-scope"', "scope_name", "new-scope"), + ("scope_version", "", 'event.scope.version = "2.0"', "scope_version", "2.0"), + ("timestamp", 'event.timestamp.sec = 2147483648; event.timestamp.nsec = 987654321', + "", "timeUnixNano", ["2147483648987654321", "2147483648987654321"]), + ("observed_timestamp", 'otlp:set("observed_timestamp", fb.uint64("2147483648987654321"))', + "", "observedTimeUnixNano", ["2147483648987654321", "2147483648987654321"]), + ("field_precedence", 'event.metadata:set("severity_text", "ROOT"); ' + 'otlp:set("severity_text", "NESTED"); ' + 'assert(fb.log_field(event, "severity_text"):value() == "NESTED")', + "", "severityText", ["NESTED", "NESTED"]), + ("field_zero", 'otlp:set("trace_flags", fb.uint64(0)); ' + 'event.metadata:set("trace_flags", fb.uint64(1)); ' + 'attrs:set("test", fb.log_field(event, "trace_flags"))', "", "attrs", [0, 0]), + ("field_missing", 'assert(fb.log_field(event, "absent") == nil)', + "", "attrs", [None, None]), + ("signed_min", 'attrs:set("test", fb.int64("-9223372036854775808"))', + "", "attrs", [-9223372036854775808, -9223372036854775808]), + ("signed_max", 'attrs:set("test", fb.int64("9223372036854775807"))', + "", "attrs", [9223372036854775807, 9223372036854775807]), + ("empty_array", 'attrs:set("test", fb.array())', "", "attrs", [[], []]), + ("empty_map", 'attrs:set("test", fb.map())', "", "attrs", [{}, {}]), + ("binary", 'attrs:set("test", fb.bytes("a\\0b"))', "", "bytes", ["YQBi", "YQBi"]), + ("nested_array", 'local a = fb.array(); local m = fb.map(); m:set("nested", true); ' + 'a:append(m); a:append(fb.int64(7)); attrs:set("test", a)', + "", "attrs", [[{"nested": True}, 7], [{"nested": True}, 7]]), + ("rollback", 'body:set("text", "partial"); attrs:set("test", "partial"); error("matrix-rollback")', + "", "attrs", [None, None]), + ("context_rollback", "", 'event.resource.attributes:set("test", "partial"); error("matrix-context-rollback")', + "resource", None), + ("drop_first", 'if body:get_string("text") == "operationA" then return fb.DROP end', + "", "drop", [True, False]), + ("drop_both", 'return fb.DROP', "", "drop", [True, True]), +] + + +# Operational recipes: producer enrichment stays isolated; request identity remains +# on the event. Parse/normalize helpers remove the cache/type plumbing from policies. +POLICIES.extend([ + ("producer_from_record", 'event.resource.attributes:set("service.name", body:get_string("text"))', + "", "producer", ["operationA", "operationB"]), + ("scope_from_record", 'event.scope.name = body:get_string("text")', + "", "scope_name_record", ["operationA", "operationB"]), + ("tenant_on_event", 'attrs:set("tenant.id", body:get_string("text"))', + "", "tenant", ["operationA", "operationB"]), + ("named_severity", 'fb.set_severity(event, "Warning")', + "", "named_severity", [13, 13]), + ("fractional_time", 'event.timestamp = fb.parse_time("2024-11-19T18:05:24.0300402+00:00")', + "", "timeUnixNano", ["1732039524030040200", "1732039524030040200"]), + ("correlation", 'fb.set_trace_context(event, "abcdef0123456789abcdef0123456789", "abcdef0123456789")', + "", "correlation", [True, True]), + ("json_normalize", 'local parsed = fb.parse_json(\'{"Level":"Information","Properties":{"customer":"acme"}}\'); ' + 'fb.set_severity(event, parsed:get_string("Level")); ' + 'attrs:set("customer", parsed:get("Properties"):get("customer"))', + "", "json_normalize", ["acme", "acme"]), + ("recursive_privacy", 'local parsed = fb.parse_json(\'{"Authorization":"bearer","nested":{"password":"secret"},' + '"headers":[{"AUTHORIZATION":"token"}]}\'); ' + 'fb.redact(parsed, {"authorization", "password"}); body:set("sanitized", parsed)', + "", "privacy", [True, True]), + ("flags_preserve", 'otlp:set("trace_flags", fb.uint64(255))', + "", "flags", [255, 255]), + ("event_name", 'otlp:set("event_name", "policy.event")', + "", "eventName", ["policy.event", "policy.event"]), + ("dropped_count", 'otlp:set("dropped_attributes_count", fb.uint64(0))', + "", "droppedAttributesCount", [0, 0]), +]) + +# Boundary/failure cases supplement the OTTL-inspired policy families. Each +# failure happens after a mutation, so the output must prove full rollback. +INVALID_OPERATIONS = { + "signed_overflow": 'fb.int64("9223372036854775808")', + "signed_underflow": 'fb.int64("-9223372036854775809")', + "unsigned_overflow": 'fb.uint64("18446744073709551616")', + "unsigned_negative": 'fb.uint64("-1")', + "integer_whitespace": 'fb.int64(" 1")', + "integer_plus": 'fb.int64("+1")', + "integer_empty": 'fb.int64("")', + "integer_nul": 'fb.int64("1\\0")', + "integer_fraction": 'fb.int64(1.5)', + "integer_unsafe_number": 'fb.uint64(9007199254740992)', + "integer_nan": 'fb.int64(0/0)', + "unsafe_number_read": 'fb.uint64("18446744073709551615"):number()', + "nonfinite_write": 'body:set("bad", math.huge)', + "nil_write": 'body:set("bad", nil)', + "table_write": 'body:set("bad", {})', + "map_numeric_key": 'body:get(1)', + "map_numeric_set_key": 'body:set(1, "bad")', + "scalar_get": 'body:get("text"):get("bad")', + "scalar_set": 'body:get("text"):set("bad", "bad")', + "scalar_len": 'body:get("text"):len()', + "scalar_keys": 'body:get("text"):keys()', + "get_string_wrong_type": 'body:get_string("metadata")', + "get_bool_wrong_type": 'body:get_bool("text")', + "get_number_wrong_type": 'body:get_number("text")', + "ensure_map_conflict": 'body:ensure_map("text")', + "bytes_nonstring": 'fb.bytes(3)', + "map_constructor_args": 'fb.map({})', + "array_constructor_args": 'fb.array({})', + "empty_array_index": 'fb.array():get(1)', + "array_index_zero": 'local a = fb.array(); a:append(true); a:get(0)', + "array_index_fraction": 'local a = fb.array(); a:append(true); a:get(1.5)', + "array_index_string": 'local a = fb.array(); a:append(true); a:get("1")', + "array_index_nan": 'local a = fb.array(); a:append(true); a:get(0/0)', + "array_index_infinity": 'local a = fb.array(); a:append(true); a:get(math.huge)', + "array_set_out_of_bounds": 'fb.array():set(1, true)', + "map_append": 'body:append(true)', + "body_root_array": 'event.body = fb.array()', + "metadata_root_array": 'event.metadata = fb.array()', + "timestamp_negative": 'event.timestamp.sec = -1', + "timestamp_overflow": 'event.timestamp.sec = 4294967296', + "timestamp_fraction": 'event.timestamp.sec = 1.5', + "timestamp_nan": 'event.timestamp.sec = 0/0', + "timestamp_nsec_negative": 'event.timestamp.nsec = -1', + "timestamp_nsec_overflow": 'event.timestamp.nsec = 1000000000', + "timestamp_nsec_fraction": 'event.timestamp.nsec = 0.5', + "timestamp_string": 'event.timestamp.sec = "1"', + "timestamp_wrong_shape": 'event.timestamp = {}', + "nonstring_error": 'error({message="typed error"})', + "invalid_return": 'do return "invalid" end', + "stale_view": 'local child = body:get("metadata"):get("uid"); ' + 'body:remove("metadata"); child:value()', +} +INVALID_OPERATIONS.update({ + "json_duplicate": 'fb.parse_json(\'{"secret":"one","secret":"two"}\')', + "json_trailing": 'fb.parse_json(\'{"ok":1} trailing\')', + "json_overflow": 'fb.parse_json("18446744073709551616")', + "json_invalid": 'fb.parse_json("{broken")', + "severity_unknown": 'fb.set_severity(event, "unexpected")', + "severity_double": 'otlp:set("severity_number", 9)', + "flags_bounds": 'otlp:set("trace_flags", fb.uint64(256))', + "severity_bounds": 'otlp:set("severity_number", fb.int64(25))', + "trace_bad_length": 'fb.set_trace_context(event, "abc", "abcdef0123456789")', + "trace_all_zero": 'fb.set_trace_context(event, string.rep("0",32), "abcdef0123456789")', + "trace_wrong_type": 'otlp:set("trace_id", "abcdef0123456789abcdef0123456789")', + "time_missing_zone": 'fb.parse_time("2024-11-19T18:05:24")', + "time_bad_leap": 'fb.parse_time("2023-02-29T12:00:00Z")', + "time_bad_fraction": 'fb.parse_time("2024-02-29T12:00:00.1234567890Z")', + "time_bad_zone": 'fb.parse_time("2024-02-29T12:00:00+24:00")', + "time_bad_calendar": 'fb.parse_time("2024-02-30T12:00:00Z")', + "scope_invalid_shape": 'event.context.attributes:set("scope", true)', + "scope_invalid_name": 'event.context.attributes:get("scope"):set("name", fb.int64(1))', + "resource_invalid_attrs": 'event.context.attributes:get("resource"):set("attributes", false)', + "raw_time_conflict": 'otlp:set("timestamp", fb.uint64(1))', + "zero_trace": 'otlp:set("trace_id", fb.bytes(string.rep(string.char(0),16)))', + "invalid_utf8": 'attrs:set("bad", string.char(255))', + "invalid_utf8_key": 'attrs:set(string.char(255), "bad")', + "overlong_utf8": 'attrs:set("bad", string.char(192,128))', + "surrogate_utf8": 'attrs:set("bad", string.char(237,160,128))', + "otel_unsigned_overflow": 'attrs:set("bad", fb.uint64("18446744073709551615"))', + "otel_nul_string": 'attrs:set("bad", "a" .. string.char(0) .. "b")', + "otel_nul_key": 'attrs:set("a" .. string.char(0) .. "b", "bad")', + "root_flags_invalid": 'otlp:remove("trace_flags"); event.metadata:set("trace_flags", "bad")', + "event_name_invalid": 'otlp:set("event_name", fb.int64(1))', + "context_metadata_gc": 'local replacement=fb.map(); for _,k in ipairs(event.context.metadata:keys()) do replacement:set(k,event.context.metadata:get(k)) end; event.context.metadata=replacement; collectgarbage()', + "context_replace": 'event.context = {}', + "context_root_replace": 'event.context.attributes = fb.map()', + "resource_alias_replace": 'event.resource = {}', + "transport_id_edit": 'event.context.metadata:set("resource_id", fb.int64(9))', +}) +POLICIES.extend(("bad_" + name, 'body:set("text", "partial"); attrs:set("test", "partial"); ' + code, + "", "attrs", [None, None]) for name, code in INVALID_OPERATIONS.items()) + + +def scalar(value): + if "stringValue" in value: + return value["stringValue"] + if "intValue" in value: + return int(value["intValue"]) + if "doubleValue" in value: + return value["doubleValue"] + if "boolValue" in value: + return value["boolValue"] + if "bytesValue" in value: + return value["bytesValue"] + if "arrayValue" in value: + return [scalar(item) for item in value["arrayValue"].get("values", [])] + if "kvlistValue" in value: + return attributes(value["kvlistValue"].get("values", [])) + return None + + +def attributes(items): + return {item["key"]: scalar(item["value"]) for item in items} + + +def any_value(value): + if isinstance(value, str): + return {"stringValue": value} + if isinstance(value, dict): + return {"kvlistValue": {"values": kvpairs(value)}} + raise AssertionError(f"unsupported fixture value: {value!r}") + + +def kvpairs(values): + return [{"key": key, "value": any_value(value)} for key, value in values.items()] + + +def policy_code(): + definitions = [] + for name, log_code, context_code, _, _ in POLICIES: + definitions.append(f'logs[{json.dumps(name)}] = function(event)\n' + 'local body = event.body; local attrs = event.attributes; ' + 'local otlp = event.metadata:get("otlp")\n' + log_code + '\n' + context_code + '\nend\n') + return 'local fb = require("fluentbit"); local logs = {}\n' + "\n".join(definitions) + ''' +function process_log(event) + if event.body:get_string("text") == "control" then return end + local name = event.resource.attributes:get_string("policy.case") + return logs[name](event) +end +''' + + +class PolicyRunner: + def __init__(self, manager, listener_port): + self.manager = manager + self.listener_port = listener_port + self.sequence = 0 + + def wait_for_condition(self, predicate, timeout=30): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = predicate() + if value: + return value + time.sleep(0.1) + raise AssertionError("timed out waiting for exported policy batch") + + def batches(self): + text = Path(self.manager.log_file).read_text() + return [json.loads(line) for line in text.splitlines(keepends=True) + if line.startswith("{") and line.endswith("\n")] + + def payload(self, name, request): + logs = [] + for i, text in enumerate(["operationA", "operationB", "control"]): + attrs = {"request.id": request, "http.method": "get", "http.path": "/health", + "http.url": "http://localhost/health", "flags": "A|B|C" if i == 0 else "C|D", + "total.string": "123456789" if i == 0 else "345678"} + body = {"text": text, "snake": "operation_a" if i == 0 else "operation_b", + "metadata": {"uid": "original"}} + logs.append({"timeUnixNano": "2147483648123456789", + "observedTimeUnixNano": "2147483648234567890", + "severityNumber": 9, "severityText": "INFO", "flags": 1 if i == 0 else 2, + + "eventName": "application.event", + "droppedAttributesCount": 1 if i == 0 else 2, + "traceId": "0102030405060708090a0b0c0d0e0f10", "spanId": "0102030405060708", + "attributes": kvpairs(attrs), "body": any_value(body)}) + return {"resourceLogs": [{ + "schemaUrl": "https://schema.example/original-resource", + "resource": {"attributes": kvpairs({"host.name": "localhost", "policy.case": name, + "policy.request": request})}, + "scopeLogs": [{"scope": {"name": "scope", "version": "1", "attributes": []}, + "schemaUrl": "https://schema.example/original-scope", "logRecords": logs}], + }]} + + def send(self, payload): + response = requests.post(f"http://127.0.0.1:{self.listener_port}/v1/logs", + json=payload, timeout=15) + assert response.status_code in (200, 201), response.text + + def collect(self, request): + found = [] + for batch in self.batches(): + for resource in batch["resourceLogs"]: + if attributes(resource["resource"]["attributes"]).get("policy.request") == request: + found.append(resource) + if any(scalar(log["body"]).get("text") == "control" + for resource in found for scope in resource["scopeLogs"] for log in scope["logRecords"]): + return found + return None + + def run(self, name): + self.sequence += 1 + request = f"{name}-{self.sequence}" + payload = self.payload(name, request) + log_offset = len(Path(self.manager.log_file).read_text()) + self.send(payload) + found = self.wait_for_condition(lambda: self.collect(request), timeout=30) + if not name.startswith("bad_") and name not in ("rollback", "context_rollback"): + diagnostics = Path(self.manager.log_file).read_text()[log_offset:] + assert "cannot construct" not in diagnostics + assert "policy failed" not in diagnostics + return payload["resourceLogs"][0], found + + +@pytest.fixture(scope="module", params=["input", "output"]) +def runner(request, tmp_path_factory): + config = tmp_path_factory.mktemp("typed-policies") / "policies.yaml" + processor = {"name": "fts", "code": policy_code(), "call": "process_log", + "error_mode": "keep"} + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + (source if request.param == "input" else output)["processors"] = {"logs": [processor]} + config.write_text(yaml.safe_dump({ + "service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}, + })) + # A module-owned manager survives the suite's per-test active-service + # cleanup. Fixture finalization still performs strict memory verification. + listener_port = find_available_port() + with pytest.MonkeyPatch.context() as env: + env.setenv("FLUENT_BIT_TEST_LISTENER_PORT", str(listener_port)) + env.setenv("FLUENT_BIT_HTTP_MONITORING_PORT", "0") + manager = FluentBitManager(str(config)) + try: + manager.start() + yield PolicyRunner(manager, listener_port) + finally: + manager.stop() + + +@pytest.mark.parametrize("policy", POLICIES, ids=[case[0] for case in POLICIES]) +def test_log_policy_matrix(runner, policy): + name, _, _, target, expected = policy + original, resources = runner.run(name) + if name.startswith("bad_"): + assert "edits rolled back" in Path(runner.manager.log_file).read_text() + pairs = [(resource, scope, log) for resource in resources for scope in resource["scopeLogs"] + for log in scope["logRecords"]] + logs = [log for _, _, log in pairs] + expected_count = 3 - sum(expected) if target == "drop" else 3 + assert len(logs) == expected_count + original_logs = original["scopeLogs"][0]["logRecords"] + kept_indices = [i for i in range(3) if target != "drop" or i == 2 or not expected[i]] + for i, (resource, scope, log) in zip(kept_indices, pairs): + resource_attrs = attributes(resource["resource"]["attributes"]) + scope_attrs = attributes(scope["scope"].get("attributes", [])) + if target == "resource": + assert resource_attrs.get("test") == (expected if i < 2 else None) + if target == "scope": + assert scope_attrs.get("test") == (expected if i < 2 else None) + if target == "resource_schema": + assert resource["schemaUrl"] == (expected if i < 2 else original["schemaUrl"]) + if target == "scope_schema": + assert scope["schemaUrl"] == (expected if i < 2 else original["scopeLogs"][0]["schemaUrl"]) + if target in ("scope_name", "scope_version"): + key = target.split("_")[1] + assert scope["scope"][key] == (expected if i < 2 else original["scopeLogs"][0]["scope"][key]) + original_log = original_logs[i] + if target == "producer": + assert resource_attrs.get("service.name") == (expected[i] if i < 2 else None) + if target == "scope_name_record": + assert scope["scope"]["name"] == (expected[i] if i < 2 else "scope") + baseline_attrs = attributes(original_log["attributes"]) + actual_attrs = attributes(log["attributes"]) + baseline_body = scalar(original_log["body"]) + actual_body = scalar(log["body"]) + wanted_attrs = copy.deepcopy(baseline_attrs) + wanted_body = copy.deepcopy(baseline_body) + if i < 2: + value = expected[i] if isinstance(expected, list) else expected + if target in ("attrs", "bytes") and value is not None: + wanted_attrs["test"] = value + elif target == "http.method": + wanted_attrs["http.method"] = value + elif target == "rename": + wanted_attrs["url"] = wanted_attrs.pop("http.url") + elif target == "delete" and value: + wanted_attrs.pop("http.url") + elif target == "delete_matching": + wanted_attrs.pop("http.method") + wanted_attrs.pop("http.path") + elif target == "keep": + wanted_attrs = {k: v for k, v in wanted_attrs.items() if k in ("http.method", "request.id")} + elif target == "empty": + wanted_attrs = {"request.id": wanted_attrs["request.id"]} + elif target == "merge_insert": + wanted_attrs["json_test"] = value + elif target == "merge_upsert": + wanted_attrs.update({"http.method": "post", "new": value}) + elif target == "nested": + wanted_body["metadata"]["uid"] = value + elif target == "tenant": + wanted_attrs["tenant.id"] = value + assert "tenant.id" not in resource_attrs + elif target == "json_normalize": + wanted_attrs["customer"] = value + elif target == "privacy": + wanted_body["sanitized"] = {"Authorization": "[REDACTED]", + "nested": {"password": "[REDACTED]"}, "headers": [{"AUTHORIZATION": "[REDACTED]"}]} + + assert actual_attrs == wanted_attrs + assert actual_body == wanted_body + for field in ("timeUnixNano", "observedTimeUnixNano", "severityText", "severityNumber", + "traceId", "spanId", "flags", "droppedAttributesCount", "eventName"): + wanted = expected[i] if target == field and i < 2 else original_log[field] + if name == "field_zero" and field == "flags" and i < 2: + wanted = 0 + if target == "flags" and field == "flags" and i < 2: + wanted = expected[i] + if target == "named_severity" and i < 2: + if field == "severityNumber": wanted = 13 + if field == "severityText": wanted = "Warning" + if target == "correlation" and i < 2: + if field == "traceId": wanted = "abcdef0123456789abcdef0123456789" + if field == "spanId": wanted = "abcdef0123456789" + if target == "json_normalize" and i < 2: + if field == "severityNumber": wanted = 9 + if field == "severityText": wanted = "Information" + assert log[field] == wanted, (name, field, log[field], wanted) + # The final control record proves that the batch was flushed, including + # policies that drop every selected record or fail their callback. + assert scalar(logs[-1]["body"])["text"] == "control" + + +def test_concurrent_requests_preserve_context(runner): + payloads = [runner.payload("resource_set", f"concurrent-{i}") for i in range(8)] + with ThreadPoolExecutor(max_workers=4) as executor: + list(executor.map(runner.send, payloads)) + for i in range(8): + resources = runner.wait_for_condition(lambda: runner.collect(f"concurrent-{i}"), timeout=30) + pairs = [(attributes(resource["resource"]["attributes"]), scalar(log["body"])) + for resource in resources for scope in resource["scopeLogs"] for log in scope["logRecords"]] + assert [body["text"] for _, body in pairs] == ["operationA", "operationB", "control"] + assert [attrs.get("test") for attrs, _ in pairs] == ["pass", "pass", None] + +@pytest.mark.parametrize("policy,field", [("producer_from_record", "resource"), ("scope_from_record", "scope")]) +def test_interleaved_producers_preserve_event_order(runner, policy, field): + request = f"interleaved-{policy}" + payload = runner.payload(policy, request) + logs = payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"] + logs.insert(2, copy.deepcopy(logs[0])) + runner.send(payload) + resources = runner.wait_for_condition(lambda: runner.collect(request)) + rows = [(resource, scope, scalar(log["body"])["text"]) + for resource in resources for scope in resource["scopeLogs"] for log in scope["logRecords"]] + assert [text for _, _, text in rows] == ["operationA", "operationB", "operationA", "control"] + for resource, scope, text in rows[:-1]: + if field == "resource": + assert attributes(resource["resource"]["attributes"])["service.name"] == text + else: + assert scope["scope"]["name"] == text + +@pytest.mark.parametrize("policy", ["producer_from_record", "scope_from_record", "named_severity", "correlation"]) +def test_protobuf_ingestion_preserves_context_and_exact_time(runner, policy): + from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + request_id = f"protobuf-{policy}" + payload = runner.payload(policy, request_id) + original = payload["resourceLogs"][0] + message = ExportLogsServiceRequest() + resource = message.resource_logs.add() + for key, value in attributes(original["resource"]["attributes"]).items(): + attr = resource.resource.attributes.add(); attr.key = key; attr.value.string_value = value + resource.schema_url = original["schemaUrl"] + scope = resource.scope_logs.add(); scope.scope.name = "scope"; scope.scope.version = "1" + scope.schema_url = original["scopeLogs"][0]["schemaUrl"] + for i, source in enumerate(original["scopeLogs"][0]["logRecords"]): + log = scope.log_records.add() + log.time_unix_nano = int(source["timeUnixNano"]) + log.observed_time_unix_nano = int(source["observedTimeUnixNano"]) + log.severity_number = 9; log.severity_text = "INFO"; log.flags = 257 + log.dropped_attributes_count = i + 1 + log.trace_id = bytes.fromhex(source["traceId"]); log.span_id = bytes.fromhex(source["spanId"]) + for key, value in scalar(source["body"]).items(): + item = log.body.kvlist_value.values.add(); item.key = key + if isinstance(value, str): item.value.string_value = value + else: + child = item.value.kvlist_value.values.add(); child.key = "uid"; child.value.string_value = "original" + for key, value in attributes(source["attributes"]).items(): + item = log.attributes.add(); item.key = key; item.value.string_value = value + response = requests.post(f"http://127.0.0.1:{runner.listener_port}/v1/logs", + data=message.SerializeToString(), headers={"Content-Type":"application/x-protobuf"}, + timeout=15) + assert response.status_code in (200, 201), response.text + resources = runner.wait_for_condition(lambda: runner.collect(request_id)) + pairs = [(resource, scope, log) for resource in resources for scope in resource["scopeLogs"] + for log in scope["logRecords"]] + assert [scalar(log["body"])["text"] for _, _, log in pairs] == ["operationA", "operationB", "control"] + for i, (resource, scope, log) in enumerate(pairs): + assert log["timeUnixNano"] == "2147483648123456789" + assert log["observedTimeUnixNano"] == "2147483648234567890" + assert log["flags"] == 1 and log["droppedAttributesCount"] == i + 1 + if policy == "producer_from_record": + assert attributes(resource["resource"]["attributes"]).get("service.name") == ( + ["operationA", "operationB"][i] if i < 2 else None) + if policy == "scope_from_record": + assert scope["scope"]["name"] == ["operationA", "operationB", "scope"][i] + assert log["severityNumber"] == (13 if policy == "named_severity" and i < 2 else 9) + assert log["traceId"] == ("abcdef0123456789abcdef0123456789" if policy == "correlation" and i < 2 + else "0102030405060708090a0b0c0d0e0f10") + + +def test_adjacent_scopes_share_resource_without_reordering(runner): + request_id = "adjacent-scopes" + payload = runner.payload("set", request_id) + scopes = payload["resourceLogs"][0]["scopeLogs"] + for name in ["second", "third"]: + scope = copy.deepcopy(scopes[0]); scope["scope"]["name"] = name; scopes.append(scope) + runner.send(payload) + resources = runner.wait_for_condition(lambda: runner.collect(request_id)) + assert len(resources) == 1 + assert [scope["scope"]["name"] for scope in resources[0]["scopeLogs"]] == ["scope", "second", "third"] + assert all(len(scope["logRecords"]) == 3 for scope in resources[0]["scopeLogs"]) diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_records.py b/tests/integration/scenarios/processor_fts/tests/test_fts_records.py new file mode 100644 index 00000000000..da7b2af6fe9 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_records.py @@ -0,0 +1,524 @@ +"""Record commits preserve policy lifetimes, rollback and chain boundaries.""" +import json +import socket +import time + +from pathlib import Path + +import pytest +import yaml + +from utils.test_service import FluentBitTestService + + +def start_service(tmp_path, code, placement, *, following=None, **options): + processors = [{"name": "fts", "code": code, **options}] + if following: + processors.append(following) + source = {"name": "forward", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", "tag": "stream"} + output = {"name": "stdout", "match": "*", "format": "json_lines"} + (source if placement == "input" else output)["processors"] = {"logs": processors} + config = tmp_path / "records.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + service.start() + return service + + +def log_text(service): + return Path(service.flb.log_file).read_text() + + +def rows(service): + return [json.loads(line) for line in log_text(service).splitlines(keepends=True) + if line.startswith("{") and line.endswith("\n")] + + +class Raw(bytes): + """Preencoded MessagePack value.""" + + +def event_time(seconds): + return Raw(b"\xd7\x00" + seconds.to_bytes(4, "big") + bytes(4)) + + +def pack(value): + """Encode fixture values without an optional MessagePack dependency.""" + if isinstance(value, Raw): + return bytes(value) + if isinstance(value, int): + return b"\xcf" + value.to_bytes(8, "big") + if isinstance(value, str): + data = value.encode("utf-8") + return b"\xdb" + len(data).to_bytes(4, "big") + data + if isinstance(value, bytes): + return b"\xc6" + len(value).to_bytes(4, "big") + value + if isinstance(value, list): + return b"\xdd" + len(value).to_bytes(4, "big") + b"".join(pack(item) for item in value) + if isinstance(value, dict): + return b"\xdf" + len(value).to_bytes(4, "big") + b"".join( + pack(key) + pack(item) for key, item in value.items()) + raise TypeError(type(value)) + + +def send(service, records): + if isinstance(records, dict): + records = [records] + packed = b"".join(pack([1700000000, record]) for record in records) + message = pack(["stream", packed, {"size": len(records)}]) + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as connection: + connection.sendall(message) + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("chain", ["direct", "filter", "native", "replacement"]) +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +def test_records_commits_and_retained_views(tmp_path, placement, chain, error_mode): + code = ''' + local fb = require("fluentbit") + local previous, stale + function process(log) + if stale then + -- Record handles expire with their callback; copy() keeps values. + assert(not pcall(function() return stale.body:get_string("text") end)) + assert(not pcall(function() stale.metadata:set("late", true) end)) + end + if previous then + assert(previous.body:get_string("text") == "committed") + previous.body:set("text", "late mutation") + previous.body:get("values"):set(1, "late mutation") + previous.metadata:set("late", true) + previous.timestamp.sec = 1 + end + assert(log.timestamp.sec ~= 1) + assert(log.metadata:get("late") == nil) + local index = log.body:get_number("index") + if REPLACE then log.body = log.body:get("nested") end + log.body:set("text", "committed") + log.body:set("exact", fb.uint64("18446744073709551615")) + log.body:set("bytes", fb.bytes("a\\0b")) + log.body:set("empty", fb.array()) + fb.redact(log.body, {"password"}) + previous = {body = log.body:copy(), metadata = log.metadata:copy(), + timestamp = log.timestamp} + stale = log + if index == 2 then error("expected item failure") end + if index == 4 then return fb.DROP end + end + '''.replace("REPLACE", "true" if chain == "replacement" else "false") + following = None + if chain == "filter": + following = {"name": "grep", "regex": "text ^(original|committed)$"} + elif chain == "native": + following = {"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"} + service = start_service(tmp_path, code, placement, following=following, error_mode=error_mode) + try: + batch = [] + for index in range(10): + body = {"index": index, "text": "original", "password": "secret", "values": ["original"]} + batch.append({**body, "nested": dict(body)}) + send(service, batch) + expected = [index for index in range(10) if index != 4 and + (error_mode == "keep" or index != 2)] + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= len(expected) + else None, timeout=45) + assert [row["index"] for row in output] == expected + for row in output: + failed = row["index"] == 2 + assert row["text"] == ("original" if failed else "committed") + assert row["values"] == ["original"] + assert row["date"] == 1700000000.0 + assert row["password"] == ("secret" if failed else "[REDACTED]") + if not failed: + assert row["exact"] == 18446744073709551615 + assert row["bytes"] == "a\0b" + assert row["empty"] == [] + if chain == "native": + assert row["following"] == "native" + diagnostics = log_text(service) + assert "cannot construct" not in diagnostics + assert diagnostics.count("log policy failed:") == 1 + assert "expected item failure" in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +def test_large_batch_edits_and_rollbacks_stay_record_local(tmp_path, placement, error_mode): + """Edits and rollbacks in a batch whose decoded records exceed the chunk value pool.""" + code = ''' + local fb = require("fluentbit") + function process(log) + local index = log.body:get_number("index") + log.body:remove("password") + log.body:get("nested"):set("state", "edited") + log.body:get("values"):append(index) + log.body:set("note", string.rep("n", index % 50 == 0 and 20000 or 8)) + if index % 97 == 5 then error("expected item failure") end + end + ''' + service = start_service(tmp_path, code, placement, error_mode=error_mode) + try: + count = 3000 + batch = [{"index": index, "password": "secret", "text": "x" * 64, + "nested": {"state": "original", "keep": index}, "values": ["first"]} + for index in range(count)] + send(service, batch) + expected = [index for index in range(count) + if error_mode == "keep" or index % 97 != 5] + output = service.wait_for_condition( + lambda: rows(service) if len(rows(service)) >= len(expected) else None, timeout=90) + assert [row["index"] for row in output] == expected + for row in output: + index = row["index"] + assert row["text"] == "x" * 64 + assert row["nested"]["keep"] == index + if index % 97 == 5: + assert row["password"] == "secret" + assert row["nested"]["state"] == "original" + assert row["values"] == ["first"] + assert "note" not in row + else: + assert "password" not in row + assert row["nested"]["state"] == "edited" + assert row["values"] == ["first", index] + assert len(row["note"]) == (20000 if index % 50 == 0 else 8) + assert "cannot construct" not in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +@pytest.mark.parametrize("chain", ["direct", "native"]) +def test_records_output_limit_rolls_back_complete_batch(tmp_path, placement, error_mode, chain): + # The native chain edits chunk values in place; failure must restore every record, + # including removed trailing keys, before the following processor runs. + code = '''local saved; function process(log) + if saved then saved:set("edited", false) end + saved = log.body:copy() + log.body:remove("text") + log.body:set("edited", true) + if log.body:get_number("index") == 599 then + log.body:set("large", string.rep("x", 70000)) + end + end''' + following = None + if chain == "native": + following = {"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"} + service = start_service(tmp_path, code, placement, following=following, + error_mode=error_mode, output_limit="65536") + try: + send(service, [{"index": index, "text": "original"} for index in range(600)]) + service.wait_for_condition(lambda: "cannot construct policy output" in log_text(service), + timeout=45) + if error_mode == "keep": + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) == 600 + else None, timeout=45) + assert [row["index"] for row in output] == list(range(600)) + assert all("edited" not in row and "large" not in row for row in output) + assert all(row["text"] == "original" for row in output) + assert all(row.get("following") == ("native" if chain == "native" else None) + for row in output) + # Stop flushes all pending output, so drop assertions cannot pass early. + service.stop() + if error_mode == "drop": + assert rows(service) == [] + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +def test_records_edit_fits_single_tree_budget(tmp_path, placement): + service = start_service(tmp_path, 'function process(log) log.body:set("edited", true) end', + placement, memory_limit="262144", error_mode="keep") + try: + send(service, {"payload": "x" * 120000}) + output = service.wait_for_condition(lambda: rows(service), timeout=45) + assert len(output) == 1 + assert output[0]["edited"] is True + assert output[0]["payload"] == "x" * 120000 + diagnostics = log_text(service) + assert "policy failed" not in diagnostics and "cannot construct" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +def test_records_depth_error_is_item_local(tmp_path, placement, error_mode): + code = '''local fb = require("fluentbit") + function process(log) + fb.redact(log.body, {"password"}) + log.body:set("edited", true) + if log.body:get_number("index") == 1 then + local nested = fb.map() + for i = 1,63 do + local parent = fb.map(); parent:set("next", nested); nested = parent + end + log.body:set("deep", nested) + end + end''' + service = start_service(tmp_path, code, placement, error_mode=error_mode) + try: + send(service, [{"index": index, "password": "secret"} for index in range(3)]) + expected = [0, 1, 2] if error_mode == "keep" else [0, 2] + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= len(expected) + else None, timeout=45) + assert [row["index"] for row in output] == expected + for row in output: + assert "deep" not in row + assert row["password"] == ("secret" if row["index"] == 1 else "[REDACTED]") + assert ("edited" in row) == (row["index"] != 1) + diagnostics = log_text(service) + assert "cannot construct policy output" not in diagnostics + assert diagnostics.count("log policy failed:") == 1 + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +def test_failed_records_do_not_poison_later_redactions(tmp_path, placement): + code = '''local fb = require("fluentbit") + function process(log) + log.body:set("temporary", string.rep("x", 4096)) + if log.body:get_number("index") % 100 ~= 99 then error("expected rejection") end + log.body:remove("temporary") + fb.redact(log.body, {"password"}) + end''' + service = start_service(tmp_path, code, placement, error_mode="drop", memory_limit="262144") + try: + send(service, [{"index": index, "password": "secret"} for index in range(2000)]) + # Wait for the complete output and its diagnostics together: slow + # memory-checker runs can emit them in separate flushes. + def ready(): + output = rows(service) + if len(output) >= 20 and "expected rejection" in log_text(service): + return output + return None + + output = service.wait_for_condition(ready, timeout=120) + assert [row["index"] for row in output] == list(range(99, 2000, 100)) + assert all(row["password"] == "[REDACTED]" and "temporary" not in row for row in output) + diagnostics = log_text(service) + assert "cannot construct" not in diagnostics + assert "memory limit" not in diagnostics + assert "expected rejection" in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +def test_empty_body_and_metadata_fork_on_first_write(tmp_path, placement): + code = '''local previous + function process(log) + if previous then previous:ensure_map("nested"):set("value", "late") end + assert(log.body:len() == 0 and log.metadata:len() == 0) + log.body:ensure_map("nested"):set("value", "body") + assert(log.metadata:len() == 0) + log.metadata:ensure_map("nested"):set("value", "metadata") + assert(log.body:get("nested"):get_string("value") == "body") + previous = log.body:copy() + end''' + service = start_service(tmp_path, code, placement) + try: + send(service, [{}, {}]) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 2 + else None, timeout=45) + assert [row["nested"] for row in output] == [{"value": "body"}, {"value": "body"}] + diagnostics = log_text(service) + assert "policy failed" not in diagnostics and "cannot construct" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("chain", ["direct", "native"]) +def test_view_cache_follows_writes(tmp_path, placement, chain): + # Views cache their resolved node; every write must invalidate that cache. + code = r'''local fb = require("fluentbit") + local stale + function process(log) + if stale then assert(not pcall(function() return stale:get_string("p") end)) end + local b = log.body + local h = b:get("h") + assert(h:get_string("p") == "x") + b:get("h"):set("p", "y") + assert(h:get_string("p") == "y") + b:remove("h") + assert(not pcall(function() return h:get_string("p") end)) + b:set("h", fb.map()) + b:get("h"):set("p", "z") + assert(h:get_string("p") == "z") + local first = b:get("values"):get(1) + b:get("values"):set(1, "b") + assert(first:value() == "b") + local copy = b:copy() + local copied = copy:get("h") + b:get("h"):set("p", "w") + assert(copied:get_string("p") == "z") + stale = b:get("h") + end''' + following = None + if chain == "native": + following = {"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"} + service = start_service(tmp_path, code, placement, following=following) + try: + send(service, [{"index": index, "h": {"p": "x"}, "values": ["a"]} for index in range(3)]) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 3 + else None, timeout=45) + assert [row["h"] for row in output] == [{"p": "w"}] * 3 + assert [row["values"] for row in output] == [["b"]] * 3 + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("chain", ["direct", "native"]) +def test_untouched_bodies_keep_original_values(tmp_path, placement, chain): + # Plain-log bodies are built only when a policy uses them; untouched bodies + # must leave the pipeline exactly as they arrived. + code = r'''local fb = require("fluentbit") + function process(log) + local index = log.timestamp.sec % 10 + if index == 1 then log.timestamp.sec = log.timestamp.sec + 100 end + if index == 2 then log.metadata:set("seen", true) end + if index == 3 then return fb.DROP end + if index == 4 then assert(log.body:get_string("text") == "original") end + if index == 5 then log.body = log.body:get("nested") end + end''' + following = None + if chain == "native": + following = {"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"} + service = start_service(tmp_path, code, placement, following=following) + try: + records = [{"text": "original", "big": 18446744073709551615, "bytes": b"a\0b", + "nested": {"inner": "value", "list": [1, "two"]}} for _ in range(6)] + packed = b"".join(pack([1700000000 + index, record]) for index, record in enumerate(records)) + message = pack(["stream", packed, {"size": len(records)}]) + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as conn: + conn.sendall(message) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 5 + else None, timeout=45) + extra = {"following": "native"} if chain == "native" else {} + full = {"text": "original", "big": 18446744073709551615, "bytes": "a\0b", + "nested": {"inner": "value", "list": [1, "two"]}, **extra} + assert [row["date"] for row in output] == [1700000000.0, 1700000101.0, 1700000002.0, + 1700000004.0, 1700000005.0] + for position, row in enumerate(output[:4]): + row.pop("date") + # stdout renders record metadata under __internal__. + metadata = row.pop("__internal__", {}).get("log_metadata") + assert metadata == ({"seen": True} if position == 2 else None) + assert row == full + nested = output[4] + nested.pop("date") + assert nested == {"inner": "value", "list": [1, "two"], **extra} + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +NESTED_RECORD = {"top": "t", "n": {"a": {"b": "deep", "list": [{"x": 1}, [2, "three"]]}, "pw": "secret"}, + "arr": [{"pw": "s1"}, {"k": "v"}]} +NESTED_CASES = { + "read": ('assert(log.body:get("n"):get("a"):get_string("b") == "deep")', NESTED_RECORD), + "write": ('log.body:get("n"):get("a"):set("b", "changed")', + {**NESTED_RECORD, "n": {"a": {"b": "changed", "list": [{"x": 1}, [2, "three"]]}, + "pw": "secret"}}), + "redact": ('fb.redact(log.body, {"pw"})', + {**NESTED_RECORD, "n": {**NESTED_RECORD["n"], "pw": "[REDACTED]"}, + "arr": [{"pw": "[REDACTED]"}, {"k": "v"}]}), + "append": ('log.body:get("n"):get("a"):get("list"):append("x")', + {**NESTED_RECORD, "n": {"a": {"b": "deep", "list": [{"x": 1}, [2, "three"], "x"]}, + "pw": "secret"}}), + "replace": ('log.body:set("n", "flat")', {**NESTED_RECORD, "n": "flat"}), + "copy": ('log.body:set("dup", log.body:get("n"))', {**NESTED_RECORD, "dup": NESTED_RECORD["n"]}), +} + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("case", list(NESTED_CASES)) +def test_nested_values_open_on_demand(tmp_path, placement, case): + # Nested plain-log containers are built only when a policy opens them. + statement, expected = NESTED_CASES[case] + code = 'local fb = require("fluentbit")\nfunction process(log) ' + statement + ' end' + service = start_service(tmp_path, code, placement) + try: + send(service, [NESTED_RECORD, NESTED_RECORD]) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 2 + else None, timeout=45) + for row in output: + row.pop("date") + assert row == expected + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +def test_failed_chunk_encoding_does_not_leak_records(tmp_path): + """A chunk that cannot be encoded must not reach the next chunk's tag.""" + processors = [{"name": "fts", "code": "function process(log) end"}, + {"name": "content_modifier", "action": "insert", "key": "added", "value": "v"}] + source = {"name": "forward", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", + "processors": {"logs": processors}} + output = {"name": "stdout", "match": "tagtwo", "format": "json_lines"} + config = tmp_path / "encode.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + service.start() + try: + # The second record carries a millisecond timestamp outside the event time range. + first = pack([1700000000, {"msg": "one"}]) + pack([1700000000123, {"msg": "bad"}]) + second = pack([1700000001, {"msg": "two"}]) + for message in (pack(["tagone", first, {"size": 2}]), pack(["tagtwo", second, {"size": 1}])): + with socket.create_connection(("127.0.0.1", service.flb_listener_port), + timeout=30) as connection: + connection.sendall(message) + time.sleep(0.5) + service.wait_for_condition(lambda: rows(service), timeout=20, + description="tagtwo output") + time.sleep(1) + assert [row["msg"] for row in rows(service)] == ["two"] + finally: + service.stop() + + +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +def test_missing_transport_ids_are_group_local(tmp_path, error_mode): + """A group without transport IDs does not skip the policy for other groups.""" + service = start_service(tmp_path, "function process(log) log.body:set('edited', true) end", + "input", following={"name": "content_modifier", "action": "insert", + "key": "native", "value": "v"}, + error_mode=error_mode) + try: + start, end = event_time(0xFFFFFFFF), event_time(0xFFFFFFFE) + now = event_time(1700000000) + entries = [[[start, {"schema": "otlp", "resource_id": 0, "scope_id": 0}], + {"resource": {}, "scope": {"name": "good"}}], + [[now, {"otlp": {}}], {"msg": "healthy-otel"}], + [[end, {}], {}], + [[start, {"schema": "otlp"}], {"resource": {}, "scope": {"name": "bad"}}], + [[now, {"otlp": {}}], {"msg": "missing-ids"}], + [[end, {}], {}], + [[now, {}], {"msg": "healthy-plain"}]] + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as connection: + connection.sendall(pack(["stream", entries])) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 3 else None, + timeout=20, description="three records") + assert sorted(row["msg"] for row in output) == ["healthy-otel", "healthy-plain", "missing-ids"] + assert all(row["edited"] is True and row["native"] == "v" for row in output) + assert "cannot construct policy output" not in log_text(service) + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py b/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py new file mode 100644 index 00000000000..9b6051f5f77 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py @@ -0,0 +1,253 @@ +"""Privacy policies must execute for every item, despite malformed siblings.""" +import base64 +import copy +import json +import re +import time +from pathlib import Path + +import pytest +import requests +import yaml +from google.protobuf.json_format import ParseDict +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port +from test_fts_signals import attr, metric, span +from test_fts_policies import attributes, scalar + +POLICY = ''' +local fb = require("fluentbit") +if type(getmetatable("")) ~= "string" then error("string metatable exposed") end +assert = function() end +require = function() return {} end +fb._attach = function() return function() end end +fb._validate = function() end +fb._fts_wrap = function() return {} end +fb._fts_validate_telemetry = function() end +fb._fts_equal = function() return true end +math.huge = 0 +string.match = function() return true end +function process(record) + if record.body ~= nil then + fb.redact(record.body, {"password", "secret"}) + if record.body:get_string("action") == "bad" then record.metadata:ensure_map("otlp"):set("severity_number", fb.int64(999)) end + elseif record.signal == "metrics" then + for i=1,record.datapoints:len() do + fb.redact(record.datapoints:get(i):get("attributes"), {"secret", "password"}) + end + if record.name == "bad" then record.datapoints:get(1):set("time_unix_nano", fb.uint64(0)) end + else + fb.redact(record.attributes, {"secret", "password"}) + if record.name == "bad" then record.status:set("code", fb.int64(7)) end + end +end +''' + + +class BatchRunner: + def __init__(self, manager, port, signal): + self.manager, self.port, self.signal = manager, port, signal + self.sequence = 0 + self.log_offset = 0 + self.keys = {"logs": ("resourceLogs", "scopeLogs", "logRecords"), + "metrics": ("resourceMetrics", "scopeMetrics", "metrics"), + "traces": ("resourceSpans", "scopeSpans", "spans")}[signal] + + def send(self, items, transport="protobuf", scopes=None, resource_attributes=None): + self.log_offset = len(Path(self.manager.log_file).read_text()) + self.sequence += 1 + request = str(self.sequence) + resource_key, scope_key, item_key = self.keys + payload = {resource_key: [{"resource": {"attributes": [attr("test.request", request)] + (resource_attributes or [])}, + scope_key: scopes or [{"scope": {"name": "audit"}, item_key: items}]}]} + if transport == "protobuf": + def convert(value): + if isinstance(value, dict): + return {k: base64.b64encode(bytes.fromhex(v)).decode() if k in + ("traceId", "spanId", "parentSpanId") else convert(v) for k, v in value.items()} + if isinstance(value, list): + return [convert(v) for v in value] + return value + message = {"logs": ExportLogsServiceRequest, "metrics": ExportMetricsServiceRequest, + "traces": ExportTraceServiceRequest}[self.signal]() + ParseDict(convert(payload), message) + response = requests.post(f"http://127.0.0.1:{self.port}/v1/{self.signal}", + data=message.SerializeToString(), headers={"Content-Type": "application/x-protobuf"}, timeout=30) + else: + response = requests.post(f"http://127.0.0.1:{self.port}/v1/{self.signal}", json=payload, timeout=30) + assert response.status_code in (200, 201), response.text + expected = sum(len(s[item_key]) for s in scopes) if scopes else len(items) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + resources = [] + text = Path(self.manager.log_file).read_text() + # Diagnostics can interrupt a large stdout write. Remove complete log + # messages before reconstructing the JSON artifact, retaining partial writes. + text = re.sub(r"\[\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d+\] " + r"\[\s*(?:error|warn|info|debug|trace)\][^\n]*\n", "", text) + for line in text.splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + try: + batch = json.loads(line) + except json.JSONDecodeError: + continue + for resource in batch.get(resource_key, []): + if attributes(resource.get("resource", {}).get("attributes", [])).get("test.request") == request: + resources.append(resource) + rows = [item for r in resources for s in r[scope_key] for item in s[item_key]] + if len(rows) >= expected: + return resources, rows + time.sleep(0.1) + raise AssertionError(Path(self.manager.log_file).read_text()[-5000:]) + + + def recent_log(self): + return Path(self.manager.log_file).read_text()[self.log_offset:] + + +@pytest.fixture(scope="module", params=[(s, p) for s in ("logs", "metrics", "traces") + for p in ("input", "output")]) +def batch_runner(request, tmp_path_factory): + signal, placement = request.param + port = find_available_port() + source = {"name": "opentelemetry", "port": port, "buffer_max_size": "8M"} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + (source if placement == "input" else output)["processors"] = { + signal: [{"name": "fts", "code": POLICY, "error_mode": "keep", "memory_limit": "4M"}]} + config = tmp_path_factory.mktemp("fts-security-batches") / "config.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": [source], "outputs": [output]}})) + manager = FluentBitManager(str(config)) + try: + manager.start() + yield BatchRunner(manager, port, signal) + finally: + manager.stop() + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_duplicate_parent_redaction(batch_runner, transport): + if batch_runner.signal == "metrics": + item = metric("gauge", "privacy") + values = item["gauge"]["dataPoints"][0]["attributes"] + elif batch_runner.signal == "traces": + item = span("privacy") + values = item["attributes"] + else: + item = {"timeUnixNano": "2147483648123456789", "body": {"kvlistValue": {"values": []}}} + values = item["body"]["kvlistValue"]["values"] + values.extend([{"key": "a", "value": {"kvlistValue": {"values": [attr("password", text)]}}} + for text in ("first-secret", "second-secret")]) + _, rows = batch_runner.send([item], transport) + rendered = json.dumps(rows) + assert "first-secret" not in rendered and "second-secret" not in rendered + assert rendered.count("[REDACTED]") >= 2 + assert "cannot construct FTS" not in batch_runner.recent_log() + + +def test_validation_is_private_and_item_local(batch_runner): + if batch_runner.signal == "metrics": + items = [metric("gauge", "bad"), metric("gauge", "good")] + elif batch_runner.signal == "traces": + items = [span("bad"), span("good")] + else: + items = [{"severityNumber": 2, "body": {"kvlistValue": {"values": [attr("action", action), + attr("secret", "sensitive")]}}} for action in ("bad", "good")] + _, rows = batch_runner.send(items) + assert len(rows) == 2 + if batch_runner.signal == "metrics": + points = [row["gauge"]["dataPoints"][0] for row in rows] + assert points[0]["timeUnixNano"] == "2147483648123456789" + assert attributes(points[0]["attributes"])["secret"] == "sensitive" + assert attributes(points[1]["attributes"])["secret"] == "[REDACTED]" + elif batch_runner.signal == "traces": + assert rows[0]["status"]["code"] == 1 + assert attributes(rows[0]["attributes"])["secret"] == "sensitive" + assert attributes(rows[1]["attributes"])["secret"] == "[REDACTED]" + else: + assert rows[0]["severityNumber"] == 2 + assert scalar(rows[0]["body"])["secret"] == "sensitive" + assert scalar(rows[1]["body"])["secret"] == "[REDACTED]" + log = batch_runner.recent_log() + assert "edits rolled back" in log and "cannot construct FTS" not in log + + +def test_unknown_inherited_kind_does_not_bypass_privacy(batch_runner): + if batch_runner.signal != "traces": + pytest.skip("trace enum preservation") + items = [span("unknown"), span("good")] + items[0]["kind"] = 9 + items[0]["parentSpanId"] = "0" * 16 + items[0]["endTimeUnixNano"] = "1" + _, rows = batch_runner.send(items) + assert len(rows) == 2 and rows[0]["kind"] == 9 + assert all(attributes(r["attributes"])["secret"] == "[REDACTED]" for r in rows) + assert "edits rolled back" not in batch_runner.recent_log() + + +def test_large_trace_batch_is_redacted(batch_runner): + if batch_runner.signal != "traces": + pytest.skip("trace batch memory accounting") + items = [span(f"span-{i}") for i in range(4000)] + for item in items: + item["attributes"] += [attr(f"extra-{i}", "value") for i in range(3)] + resources, rows = batch_runner.send(items, resource_attributes=[attr("bulk", "b" * 32768)]) + assert len(rows) == 4000 and len(resources) == 1 + assert len(resources[0]["scopeSpans"]) == 1 + assert all(attributes(r["attributes"])["secret"] == "[REDACTED]" for r in rows) + assert "cannot construct FTS" not in batch_runner.recent_log() + + +@pytest.mark.parametrize("separate_scopes", [False, True]) +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_repeated_metric_names_are_preserved(batch_runner, separate_scopes, transport): + if batch_runner.signal != "metrics": + pytest.skip("metric family identity") + items = [metric("gauge", "same"), metric("gauge", "same")] + items[1]["gauge"]["dataPoints"][0]["asInt"] = "42" + items[0]["metadata"] = [attr("family.id", "first")] + items[1]["metadata"] = [attr("family.id", "second")] + scopes = [{"scope": {"name": name}, "metrics": [item]} for name, item in zip(("one", "two"), items)] + resources, rows = batch_runner.send(items, transport, scopes=scopes if separate_scopes else None) + assert len(rows) == 2 + assert [int(r["gauge"]["dataPoints"][0]["asInt"]) for r in rows] == [8, 42] + assert [attributes(r["metadata"])["family.id"] for r in rows] == ["first", "second"] + assert all(attributes(r["gauge"]["dataPoints"][0]["attributes"])["secret"] == "[REDACTED]" for r in rows) + if separate_scopes: + assert [s["scope"]["name"] for r in resources for s in r["scopeMetrics"]] == ["one", "two"] + assert "cannot construct FTS" not in batch_runner.recent_log() + + +def test_snapshot_limit_is_item_local(batch_runner): + if batch_runner.signal != "traces": + pytest.skip("trace snapshot isolation") + items = [span("oversized"), span("good")] + items[0]["attributes"] += [attr(f"field-{i}", "value") for i in range(9000)] + _, rows = batch_runner.send(items) + assert len(rows) == 2 + assert attributes(rows[0]["attributes"])["secret"] == "sensitive" + assert attributes(rows[1]["attributes"])["secret"] == "[REDACTED]" + text = batch_runner.recent_log() + assert "snapshot limits" in text and "cannot construct FTS" not in text + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_untouched_invalid_metric_does_not_skip_redaction(batch_runner, transport): + if batch_runner.signal != "metrics": + pytest.skip("metric field validation") + item = metric("gauge", "invalid-inherited-time") + point = item["gauge"]["dataPoints"][0] + point["startTimeUnixNano"] = "2147483648999999999" + point["timeUnixNano"] = "2147483648000000000" + _, rows = batch_runner.send([item], transport) + result = rows[0]["gauge"]["dataPoints"][0] + assert result["startTimeUnixNano"] == point["startTimeUnixNano"] + assert result["timeUnixNano"] == point["timeUnixNano"] + assert attributes(result["attributes"])["secret"] == "[REDACTED]" + assert "policy failed" not in batch_runner.recent_log() + assert "cannot construct FTS" not in batch_runner.recent_log() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_signals.py b/tests/integration/scenarios/processor_fts/tests/test_fts_signals.py new file mode 100644 index 00000000000..0c07928a52d --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_signals.py @@ -0,0 +1,471 @@ +"""FTS metrics and traces policies based on OTTL transformation workflows.""" +import base64 +import copy +import json +import time +from pathlib import Path + +import pytest +import requests +import yaml +from google.protobuf.json_format import ParseDict +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port +from test_fts_policies import attributes + + +POLICY = r''' +local fb = require("fluentbit") +function process(record) + local case = record.name + if case == "control" or case == "noop" then return end + if case == "rename" then + record.name = "normalized"; record.resource.attributes:set("service.name", "normalized") + record.scope.name = "normalized.scope"; record.scope.version = "2" + record.resource.schema_url = "https://example.com/resource" + record.scope.schema_url = "https://example.com/scope" + elseif case == "resource" then + record.resource.attributes:set("deployment.environment.name", "production") + record.resource.attributes:remove("secret") + record.scope.attributes:set("policy.version", fb.int64(2)) + elseif case == "tag" then + assert(type(record.tag) == "string" and #record.tag > 0) + record.resource.attributes:set("source.tag", record.tag) + elseif case == "module_guard" then + assert(module == nil and os == nil and debug == nil and package == nil) + assert(not pcall(function() module("os") end)) + assert(not pcall(require, "debug")) + record.name = "restricted" + elseif case == "drop" then return fb.DROP + elseif case == "rollback" then + record.name = "partial"; record.resource.attributes:set("secret", "partial") + error("rollback-test") + elseif case == "bad_name" then record.name = fb.map() + elseif case == "bad_utf8" then record.name = string.char(255) + elseif case == "bad_nul" then record.name = "a\0b" + elseif case == "bad_resource" then record.resource.attributes:set("invalid", fb.uint64("18446744073709551615")) + elseif case == "bad_scope" then record.scope.attributes = fb.array() + elseif case == "bad_return" then return true + elseif case == "budget" then while true do end + elseif record.signal == "metrics" then + if case == "metadata" then + record.description = "normalized description"; record.unit = "s" + elseif case == "labels" then + for i=1,record.datapoints:len() do + local attrs = record.datapoints:get(i):get("attributes") + attrs:set("region", "west"); attrs:remove("secret") + end + elseif case == "values" then + fb.set_metric_value(record.datapoints:get(1), fb.int64("-9223372036854775808")) + elseif case == "scale" then fb.scale_metric(record, 2, "s") + elseif case == "scale_fraction" then fb.scale_metric(record, 0.5, "s") + elseif case == "to_sum" then + record.type = "sum"; record.aggregation_temporality = "delta"; record.is_monotonic = false + elseif case == "to_gauge" then record.type = "gauge" + elseif case == "copy" then + local derived = fb.copy_metric(record, "derived", "copy", "ms") + derived.resource.attributes:set("service.name", "derived") + elseif case == "extract_count" then fb.extract_metric(record, "count", "derived") + elseif case == "extract_sum" then fb.extract_metric(record, "sum", "derived") + elseif case == "extract_avg" then fb.extract_metric(record, "avg", "derived") + elseif case:match("^aggregate_") then + fb.aggregate_metric(record, {"region"}, case:sub(11)) + elseif case == "merge_buckets" then + local bounds = fb.array(); bounds:append(2.0) + fb.merge_histogram_buckets(record, bounds) + elseif case == "bad_value" then fb.set_metric_value(record.datapoints:get(1), fb.map()) + elseif case == "typed_labels" then + local attrs = record.datapoints:get(1):get("attributes") + attrs:set("typed", fb.int64("9223372036854775807")); attrs:set("enabled", true) + attrs:set("binary", fb.bytes("\0\255")) + local nested = fb.map(); nested:set("count", fb.int64(2)); attrs:set("nested", nested) + elseif case == "bad_label" then + record.datapoints:get(1):get("attributes"):set("typed", fb.uint64("18446744073709551615")) + elseif case == "bad_time" then record.datapoints:get(1):set("time_unix_nano", fb.uint64(0)) + end + else + if case == "redact" then + fb.redact(record.attributes, {"secret"}) + fb.redact(record.events, {"secret"}); fb.redact(record.links, {"secret"}) + elseif case == "status" then record.status:set("code", fb.int64(2)); record.status:set("message", "failed") + elseif case == "kind" then record.kind = 3 + elseif case == "correlation" then + record.trace_id = "1234567890abcdef1234567890abcdef" + record.span_id = "1234567890abcdef"; record.parent_span_id = "" + elseif case == "time" then record.end_time_unix_nano = fb.uint64("2147483648999999999") + elseif case == "events" then + local events = fb.array(); local event = fb.map() + event:set("name", "normalized.event") + event:set("time_unix_nano", fb.uint64("2147483648123456789")) + event:set("attributes", fb.map()); event:set("dropped_attributes_count", fb.uint64(0)) + events:append(event); record.events = events + elseif case == "bad_id" then record.trace_id = "00000000000000000000000000000000" + elseif case == "bad_time" then record.end_time_unix_nano = fb.uint64(0) + elseif case == "bad_status" then record.status:set("code", fb.int64("9223372036854775807")) + end + end +end +''' + + +def attr(key, value): + return {"key": key, "value": {"stringValue": value}} + + +def number_point(value=8): + return {"attributes": [attr("region", "east"), attr("secret", "sensitive")], + "startTimeUnixNano": "2147483648000000000", "timeUnixNano": "2147483648123456789", + "asInt": str(value), "flags": 1} + + +def metric(kind, name): + point = number_point() + if kind == "histogram": + point.pop("asInt") + point.update(count="3", sum=4.0, bucketCounts=["1", "1", "1"], explicitBounds=[1.0, 2.0], + min=0.5, max=2.5) + elif kind == "exponentialHistogram": + point.pop("asInt") + point.update(count="3", sum=2.0, scale=0, zeroCount="1", zeroThreshold=0.0, + positive={"offset": 0, "bucketCounts": ["1"]}, + negative={"offset": 0, "bucketCounts": ["1"]}, min=-1.0, max=2.0) + elif kind == "summary": + point.pop("asInt") + point.update(count="3", sum=4.0, quantileValues=[{"quantile": 0.5, "value": 1.5}, + {"quantile": 0.9, "value": 2.5}]) + point.pop("flags") + data = {"dataPoints": [point]} + if kind in ("sum", "histogram", "exponentialHistogram"): + data["aggregationTemporality"] = 2 + if kind == "sum": + data["isMonotonic"] = True + return {"name": name, "description": "original description", "unit": "ms", kind: data} + + +def span(name): + return {"name": name, "traceId": "0102030405060708090a0b0c0d0e0f10", + "spanId": "0102030405060708" if name != "control" else "0807060504030201", + "parentSpanId": "1112131415161718", "traceState": "vendor=value", "kind": 2, + "startTimeUnixNano": "2147483648000000000", "endTimeUnixNano": "2147483648123456789", + "attributes": [attr("secret", "sensitive"), {"key": "exact", "value": {"intValue": "9223372036854775807"}}], + "events": [{"timeUnixNano": "2147483648123456789", "name": "original.event", + "attributes": [attr("secret", "sensitive")], "droppedAttributesCount": 4}], + "links": [{"traceId": "0102030405060708090a0b0c0d0e0f10", "spanId": "1112131415161718", + "traceState": "vendor=link", "attributes": [attr("secret", "sensitive")], + "droppedAttributesCount": 5}], + "status": {"code": 1, "message": "ok"}, "droppedAttributesCount": 1, + "droppedEventsCount": 2, "droppedLinksCount": 3} + + +class SignalRunner: + def __init__(self, manager, port, signal): + self.manager, self.port, self.signal = manager, port, signal + self.sequence = 0 + + def run(self, case, kind="gauge", transport="protobuf", points=None, temporality=None, + extra_resource=None): + self.log_offset = len(Path(self.manager.log_file).read_text()) + self.sequence += 1 + request_id = f"{case}-{self.sequence}" + metrics = self.signal == "metrics" + resource_key, scope_key, item_key = (("resourceMetrics", "scopeMetrics", "metrics") if metrics else + ("resourceSpans", "scopeSpans", "spans")) + items = [metric(kind, case), metric(kind, "control")] if metrics else [span(case), span("control")] + if points is not None: + items[0][kind]["dataPoints"] = points + if temporality is not None: + items[0][kind]["aggregationTemporality"] = temporality + payload = {resource_key: [{"resource": {"attributes": [attr("test.request", request_id), + attr("secret", "sensitive"), attr("service.name", "original")] + (extra_resource or []), + "droppedAttributesCount": 7}, + "schemaUrl": "https://example.com/original-resource", + scope_key: [{"scope": {"name": "original.scope", "version": "1", + "attributes": [{"key": "exact", "value": {"intValue": "9223372036854775807"}}], + "droppedAttributesCount": 6}, "schemaUrl": "https://example.com/original-scope", + item_key: items}]}]} + if transport == "json": + response = requests.post(f"http://127.0.0.1:{self.port}/v1/{self.signal}", json=payload, timeout=10) + else: + def convert(value): + if isinstance(value, dict): + return {k: base64.b64encode(bytes.fromhex(v)).decode() if k in + ("traceId", "spanId", "parentSpanId") else convert(v) for k, v in value.items()} + if isinstance(value, list): + return [convert(v) for v in value] + return value + message = ExportMetricsServiceRequest() if metrics else ExportTraceServiceRequest() + ParseDict(convert(payload), message) + response = requests.post(f"http://127.0.0.1:{self.port}/v1/{self.signal}", + data=message.SerializeToString(), headers={"Content-Type": "application/x-protobuf"}, timeout=10) + assert response.status_code in (200, 201), response.text + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + rows = [] + for line in Path(self.manager.log_file).read_text().splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + batch = json.loads(line) + for resource in batch.get(resource_key, []): + if attributes(resource.get("resource", {}).get("attributes", [])).get("test.request") != request_id: + continue + for scope in resource.get(scope_key, []): + rows += [(resource, scope, item) for item in scope.get(item_key, [])] + if rows: + if not case.startswith("bad_") and case not in ("rollback", "budget"): + diagnostics = Path(self.manager.log_file).read_text()[self.log_offset:] + assert "cannot construct" not in diagnostics + assert "policy failed" not in diagnostics + return items[0], rows + time.sleep(0.1) + raise AssertionError(f"no {self.signal} output for {case}: {Path(self.manager.log_file).read_text()[-5000:]}") + + +@pytest.fixture(scope="module", params=[("metrics", "input"), ("metrics", "output"), + ("traces", "input"), ("traces", "output")]) +def signal_runner(request, tmp_path_factory): + signal, placement = request.param + config = tmp_path_factory.mktemp(f"fts-{signal}-{placement}") / "signals.yaml" + port = find_available_port() + source = {"name": "opentelemetry", "port": port} + output = {"name": "stdout", "match": "*", "format": "otlp_json", "workers": 2} + (source if placement == "input" else output)["processors"] = { + signal: [{"name": "fts", "code": POLICY, "error_mode": "keep"}]} + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + manager = FluentBitManager(str(config)) + try: + manager.start() + yield SignalRunner(manager, port, signal) + finally: + manager.stop() + + +COMMON_CASES = ["noop", "rename", "resource", "tag", "module_guard", "drop", "rollback", "bad_name", "bad_utf8", "bad_nul", + "bad_resource", "bad_scope", "bad_return", "budget"] +METRIC_CASES = ["metadata", "labels", "typed_labels", "values", "scale", "scale_fraction", "to_sum", "to_gauge", "copy", + "extract_count", "extract_sum", "extract_avg", "bad_value", "bad_label", "bad_time"] +TRACE_CASES = ["redact", "status", "kind", "correlation", "time", "events", "bad_id", "bad_time", "bad_status"] + + +@pytest.mark.parametrize("case", list(dict.fromkeys(COMMON_CASES + METRIC_CASES + TRACE_CASES))) +def test_signal_policy_matrix(signal_runner, case): + signal = signal_runner.signal + if case not in COMMON_CASES and case not in (METRIC_CASES if signal == "metrics" else TRACE_CASES): + pytest.skip("case belongs to the other signal") + kind = "sum" if case == "to_gauge" else "histogram" if case.startswith("extract_") else "gauge" + original, rows = signal_runner.run(case, kind) + control = [(r, s, item) for r, s, item in rows if item["name"] == "control"] + assert len(control) == 1 + resource, scope, item = control[0] + assert attributes(resource["resource"]["attributes"])["service.name"] == "original" + assert scope["scope"]["name"] == "original.scope" + assert scope["scope"]["droppedAttributesCount"] == 6 + targets = [(r, s, item) for r, s, item in rows if item["name"] != "control"] + if case == "drop": + assert not targets + return + assert targets + resource, scope, item = targets[0] + if case.startswith("bad_") or case in ("rollback", "budget", "noop"): + assert item["name"] == case + assert attributes(resource["resource"]["attributes"])["service.name"] == "original" + assert scope["scope"]["name"] == "original.scope" + elif case == "rename": + assert item["name"] == "normalized" + assert attributes(resource["resource"]["attributes"])["service.name"] == "normalized" + assert scope["scope"]["name"] == "normalized.scope" and scope["scope"]["version"] == "2" + assert resource["schemaUrl"] == "https://example.com/resource" + assert scope["schemaUrl"] == "https://example.com/scope" + elif case == "resource": + attrs = attributes(resource["resource"]["attributes"]) + assert "secret" not in attrs and attrs["deployment.environment.name"] == "production" + assert attributes(scope["scope"]["attributes"])["policy.version"] == 2 + elif case == "tag": + assert attributes(resource["resource"]["attributes"])["source.tag"] + elif case == "module_guard": + assert item["name"] == "restricted" + elif signal == "metrics": + data = item.get("gauge") or item.get("sum") or item.get("histogram") + point = data["dataPoints"][0] + if case == "metadata": + assert item["description"] == "normalized description" and item["unit"] == "s" + elif case == "labels": + assert attributes(point["attributes"])["region"] == "west" + assert "secret" not in attributes(point["attributes"]) + elif case == "values": + assert int(point["asInt"]) == -(2**63) + elif case == "typed_labels": + attrs = attributes(point["attributes"]) + assert attrs["typed"] == 2**63-1 and attrs["enabled"] is True + assert attrs["nested"] == {"count": 2} + assert any(a["key"] == "binary" and a["value"]["bytesValue"] == "AP8=" + for a in point["attributes"]) + elif case in ("scale", "scale_fraction"): + assert float(point.get("asInt", point.get("asDouble"))) == (16 if case == "scale" else 4) + assert item["unit"] == "s" + elif case == "to_sum": + assert "sum" in item and item["sum"]["aggregationTemporality"] == 1 + assert not item["sum"]["isMonotonic"] + elif case == "to_gauge": + assert "gauge" in item and "sum" not in item + elif case == "copy" or case.startswith("extract_"): + derived = [(r, s, m) for r, s, m in targets if m["name"] == "derived"] + assert len(derived) == 1 + r, _, m = derived[0] + if case == "copy": + assert attributes(r["resource"]["attributes"])["service.name"] == "derived" + assert m["description"] == "copy" and m["unit"] == "ms" + else: + p = (m.get("gauge") or m.get("sum"))["dataPoints"][0] + assert float(p.get("asInt", p.get("asDouble"))) == {"extract_count": 3, "extract_sum": 4, + "extract_avg": 4/3}[case] + else: + assert attributes(item["attributes"])["exact"] == 2**63-1 + if case == "redact": + assert attributes(item["attributes"])["secret"] == "[REDACTED]" + assert attributes(item["events"][0]["attributes"])["secret"] == "[REDACTED]" + assert attributes(item["links"][0]["attributes"])["secret"] == "[REDACTED]" + elif case == "status": + assert item["status"] == {"code": 2, "message": "failed"} + elif case == "kind": + assert item["kind"] == 3 + elif case == "correlation": + assert item["traceId"] == "1234567890abcdef1234567890abcdef" + assert item["spanId"] == "1234567890abcdef" + assert item.get("parentSpanId", "") == "" + elif case == "time": + assert int(item["endTimeUnixNano"]) == 2147483648999999999 + elif case == "events": + assert len(item["events"]) == 1 and item["events"][0]["name"] == "normalized.event" + + +@pytest.mark.parametrize("kind", ["gauge", "sum", "histogram", "exponentialHistogram", "summary"]) +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_metric_types_round_trip(signal_runner, kind, transport): + if signal_runner.signal != "metrics": + pytest.skip("metrics case") + _, rows = signal_runner.run("noop", kind, transport) + target = next(item for _, _, item in rows if item["name"] == "noop") + point = target[kind]["dataPoints"][0] + assert int(point["timeUnixNano"]) == 2147483648123456789 + assert int(point["startTimeUnixNano"]) == 2147483648000000000 + if kind in ("gauge", "sum"): + assert int(point["asInt"]) == 8 + else: + assert int(point["count"]) == 3 + assert point["sum"] == (2.0 if kind == "exponentialHistogram" else 4.0) + + +@pytest.mark.parametrize("kind", ["gauge", "sum", "histogram", "exponentialHistogram", "summary"]) +# OTLP JSON input merges repeated series by labels before FTS runs, like 5.1. +@pytest.mark.parametrize("transport", ["protobuf"]) +def test_typed_series_keep_independent_metadata(signal_runner, kind, transport): + if signal_runner.signal != "metrics": + pytest.skip("metric datapoint semantics") + points = [copy.deepcopy(metric(kind, "noop")[kind]["dataPoints"][0]) for _ in range(2)] + points[0]["attributes"].append({"key": "typed", "value": {"intValue": "1"}}) + points[1]["attributes"].append(attr("typed", "1")) + if kind != "summary": + points[0]["flags"], points[1]["flags"] = 7, 3 + _, rows = signal_runner.run("noop", kind, transport, points) + target = next(item for _, _, item in rows if item["name"] == "noop") + exported = target[kind]["dataPoints"] + assert len(exported) == 2 + assert [attributes(point["attributes"])["typed"] for point in exported] == [1, "1"] + if kind != "summary": + assert [point["flags"] for point in exported] == [7, 3] + assert not any(entry["key"] in {"resource_index", "scope_index", "scope_flat_index"} + for entry in target.get("metadata", [])) + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_bytes_context_attributes_round_trip(signal_runner, transport): + raw = {"key": "binary", "value": {"bytesValue": base64.b64encode(b"\x00\x01\xff").decode()}} + _, rows = signal_runner.run("noop", "gauge", transport, extra_resource=[raw]) + resource = rows[0][0]["resource"]["attributes"] + exported = next(entry["value"] for entry in resource if entry["key"] == "binary") + assert base64.b64decode(exported["bytesValue"]) == b"\x00\x01\xff" + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_repeated_unlabeled_points_keep_last_value(signal_runner, transport): + """Like 5.1, a repeated label-less series keeps its last value.""" + if signal_runner.signal != "metrics": + pytest.skip("metric datapoint semantics") + points = [number_point(1), number_point(2)] + for index, point in enumerate(points): + point["attributes"] = [] + point["timeUnixNano"] = str(2147483648123456789 + index) + _, rows = signal_runner.run("noop", "gauge", transport, points) + target = next(item for _, _, item in rows if item["name"] == "noop") + assert [int(point["asInt"]) for point in target["gauge"]["dataPoints"]] == [2] + + +@pytest.mark.parametrize("operation,expected", [("sum", 20), ("min", 8), ("max", 12), + ("mean", 10), ("count", 2)]) +def test_metric_cardinality_aggregation(signal_runner, operation, expected): + if signal_runner.signal != "metrics": + pytest.skip("metric aggregation") + points = [number_point(8), number_point(12)] + for index, point in enumerate(points): + point["attributes"].append(attr("method", ["GET", "POST"][index])) + _, rows = signal_runner.run(f"aggregate_{operation}", "gauge", points=points) + target = next(item for _, _, item in rows if item["name"] != "control") + exported = target["gauge"]["dataPoints"] + assert len(exported) == 1 + assert float(exported[0].get("asInt", exported[0].get("asDouble"))) == expected + assert attributes(exported[0]["attributes"]) == {"region": "east"} + + +def test_histogram_bucket_merge(signal_runner): + if signal_runner.signal != "metrics": + pytest.skip("metric bucket semantics") + _, rows = signal_runner.run("merge_buckets", "histogram") + target = next(item for _, _, item in rows if item["name"] != "control") + point = target["histogram"]["dataPoints"][0] + assert point["explicitBounds"] == [2.0] + assert [int(value) for value in point["bucketCounts"]] == [2, 1] + assert int(point["count"]) == 3 and point["sum"] == 4.0 + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_scaling_includes_exemplars(signal_runner, transport): + if signal_runner.signal != "metrics": + pytest.skip("metric scaling") + point = number_point() + point["exemplars"] = [{"asDouble": 3.0, "timeUnixNano": point["timeUnixNano"]}] + _, rows = signal_runner.run("scale", transport=transport, points=[point]) + target = next(item for _, _, item in rows if item["name"] == "scale") + result = target["gauge"]["dataPoints"][0] + assert int(result["asInt"]) == 16 + assert result["exemplars"][0]["asDouble"] == 6.0 + + +# OTLP JSON input merges repeated series by labels before FTS runs, like 5.1. +@pytest.mark.parametrize("transport", ["protobuf"]) +def test_aggregation_preserves_distinct_start_times(signal_runner, transport): + if signal_runner.signal != "metrics": + pytest.skip("metric windows") + points = [number_point(8), number_point(42)] + points[0]["startTimeUnixNano"] = "2147483647000000000" + _, rows = signal_runner.run("aggregate_sum", transport=transport, points=points) + target = next(item for _, _, item in rows if item["name"] == "aggregate_sum") + result = target["gauge"]["dataPoints"] + assert len(result) == 2 + assert [p["startTimeUnixNano"] for p in result] == [p["startTimeUnixNano"] for p in points] + assert [int(p["asInt"]) for p in result] == [8, 42] + assert all(attributes(p["attributes"]) == {"region": "east"} for p in result) + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_extraction_preserves_unspecified_temporality(signal_runner, transport): + if signal_runner.signal != "metrics": + pytest.skip("metric extraction") + _, rows = signal_runner.run("extract_sum", "histogram", transport=transport, temporality=0) + target = next(item for _, _, item in rows if item["name"] == "derived") + assert target["sum"].get("aggregationTemporality", 0) == 0 + assert target["sum"]["dataPoints"][0]["asDouble"] == 4.0 diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_signals_pipeline.py b/tests/integration/scenarios/processor_fts/tests/test_fts_signals_pipeline.py new file mode 100644 index 00000000000..064a21806f4 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_signals_pipeline.py @@ -0,0 +1,119 @@ +"""FTS metric/span policies through protobuf export, fan-out, retries and replay.""" +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +import requests +import yaml +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from utils.test_service import FluentBitTestService +from test_fts_signals import metric, span, attr +from test_fts_policies import attributes + + +@pytest.mark.parametrize("signal", ["metrics", "traces"]) +@pytest.mark.parametrize("filesystem,replay", [(False, False), (True, False), (True, True)]) +def test_signal_route_isolation_retry_and_backlog(tmp_path, signal, filesystem, replay): + message_type = ExportMetricsServiceRequest if signal == "metrics" else ExportTraceServiceRequest + attempts, received = [], [] + blocked = {"value": replay} + + class Collector(BaseHTTPRequestHandler): + def do_POST(self): + data = self.rfile.read(int(self.headers["Content-Length"])) + attempts.append(data) + message = message_type.FromString(data) + status = 503 if blocked["value"] or len(attempts) == 1 else 200 + if status == 200: + received.append(message) + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Collector) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + policy = '''local fb = require("fluentbit") +function process(record) + if record.name == "control" then return end + record.name = "normalized" + record.resource.attributes:set("service.name", "normalized") + record.scope.name = "normalized.scope" + if record.signal == "metrics" then + record.datapoints:get(1):get("attributes"):set("exact", fb.int64("9223372036854775807")) + else + record.attributes:remove("secret") + record.attributes:set("exact", fb.int64("9223372036854775807")) + end +end''' + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + service_config = {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}", + "scheduler.base": 1, "scheduler.cap": 1} + if filesystem: + source["storage.type"] = "filesystem" + service_config["storage.path"] = str(tmp_path / "storage") + config = tmp_path / "pipeline.yaml" + config.write_text(yaml.safe_dump({"service": service_config, "pipeline": { + "inputs": [source], "outputs": [ + {"name": "stdout", "match": "*", "format": "otlp_json"}, + {"name": "opentelemetry", "match": "*", "host": "127.0.0.1", "port": server.server_port, + "retry_limit": False, "processors": {signal: [{"name": "fts", "code": policy}]}} + ]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + metrics = signal == "metrics" + resource_key, scope_key, item_key = (("resourceMetrics", "scopeMetrics", "metrics") if metrics else + ("resourceSpans", "scopeSpans", "spans")) + items = [metric("gauge", "target"), metric("gauge", "control")] if metrics else [span("target"), span("control")] + payload = {resource_key: [{"resource": {"attributes": [attr("service.name", "original")], + "droppedAttributesCount": 7}, scope_key: [{"scope": {"name": "original.scope", + "droppedAttributesCount": 6}, item_key: items}]}]} + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/{signal}", json=payload, timeout=15) + assert response.status_code in (200, 201), response.text + if replay: + service.wait_for_condition(lambda: attempts, timeout=30) + service.stop() + original_log = Path(service.flb.log_file).read_text() + assert list((tmp_path / "storage").rglob("*.flb")) + blocked["value"] = False + service = FluentBitTestService(str(config)) + service.start() + service.wait_for_condition(lambda: received, timeout=30) + service.stop() + if not replay: + original_log = Path(service.flb.log_file).read_text() + assert len(attempts) >= 2 and attempts[0] == attempts[1] + rows = [(resource, scope, item) for message in received + for resource in getattr(message, "resource_metrics" if metrics else "resource_spans") + for scope in getattr(resource, "scope_metrics" if metrics else "scope_spans") + for item in getattr(scope, "metrics" if metrics else "spans")] + assert [item.name for _, _, item in rows] == ["normalized", "control"] + for resource, scope, item in rows: + values = {entry.key: entry.value.string_value for entry in resource.resource.attributes} + assert values["service.name"] == ("normalized" if item.name == "normalized" else "original") + assert scope.scope.name == ("normalized.scope" if item.name == "normalized" else "original.scope") + assert resource.resource.dropped_attributes_count == 7 + assert scope.scope.dropped_attributes_count == 6 + if item.name == "normalized": + attrs = item.gauge.data_points[0].attributes if metrics else item.attributes + assert next(entry.value.int_value for entry in attrs if entry.key == "exact") == 2**63-1 + batches = [json.loads(line) for line in original_log.splitlines() if line.startswith("{")] + raw = [(resource, scope, item) for batch in batches for resource in batch[resource_key] + for scope in resource[scope_key] for item in scope[item_key]] + assert [item["name"] for _, _, item in raw] == ["target", "control"] + assert all(attributes(resource["resource"]["attributes"])["service.name"] == "original" + and scope["scope"]["name"] == "original.scope" for resource, scope, _ in raw) + finally: + service.stop() + server.shutdown() + server.server_close() + thread.join(timeout=5) From 57882e6ba60a95710b881325ed7ebab7c7283e23 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 18:31:36 -0600 Subject: [PATCH 19/61] lib: cfl: pack arena objects and refine string classes Arena objects use their natural alignment instead of the long double alignment, owned strings use finer size classes, and external arena allocations are aligned. Decoded record batches use about 30 percent less arena memory and 20 percent less CPU. The patch is kept isolated so it can be sent to fluent/cfl. Signed-off-by: Eduardo Silva --- lib/cfl/ARENA.md | 15 +- lib/cfl/CHANGELOG.md | 9 + lib/cfl/benchmarks/CMakeLists.txt | 3 + lib/cfl/benchmarks/README.md | 17 + lib/cfl/benchmarks/records.c | 195 ++++++++++ lib/cfl/src/cfl_arena.c | 233 ++++++++++-- lib/cfl/src/cfl_arena_internal.h | 13 + lib/cfl/src/cfl_array.c | 8 +- lib/cfl/src/cfl_kvlist.c | 2 +- lib/cfl/src/cfl_sds.c | 7 +- lib/cfl/tests/arena.c | 614 +++++++++++++++++++++++++++++- 11 files changed, 1071 insertions(+), 45 deletions(-) create mode 100644 lib/cfl/benchmarks/records.c diff --git a/lib/cfl/ARENA.md b/lib/cfl/ARENA.md index 3409f98900f..b41cdf2ecc4 100644 --- a/lib/cfl/ARENA.md +++ b/lib/cfl/ARENA.md @@ -259,8 +259,19 @@ Setting the limit to zero disables external-buffer caching. Reducing the limit immediately trims the cache. A larger limit can improve throughput for repeated large payloads but may increase retained memory and RSS. -The arena also maintains size classes for commonly-sized owned SDS values. -Callers do not need to select a class. +The arena also maintains size classes for owned SDS values up to 1 KiB. +Callers do not need to select a class. Classes below 128 bytes grow in +alignment-sized steps of the complete allocation, so short strings such as map +keys use at most one alignment unit more than they need; larger classes grow +by about 25 percent. A released string is recycled for the next string of the +same class. `cfl_sds_alloc()` reports the class capacity, which can exceed the +requested length. + +Variants, kvpairs, key/value lists, arrays, and SDS headers are aligned for the +pointers, sizes, 64-bit integers, and doubles they contain. This can be less +strict than the raw allocation alignment, which also covers `long double`, so +CFL objects pack more densely in arena chunks. Raw allocations keep the public +alignment guarantee. ## Memory statistics diff --git a/lib/cfl/CHANGELOG.md b/lib/cfl/CHANGELOG.md index 37b1f89347f..30e65835d8d 100644 --- a/lib/cfl/CHANGELOG.md +++ b/lib/cfl/CHANGELOG.md @@ -6,6 +6,15 @@ This file records the notable changes in each CFL release. - Added public request-lifetime arena allocation, duplication helpers, allocator callbacks, and optional bounded geometric chunk growth. +- Reduced arena memory for CFL object graphs: objects use their natural + alignment instead of the raw allocation alignment, and owned SDS strings use + finer size classes. Decoded log records use 27 to 31 percent less arena + memory. Owned SDS blocks are no longer zeroed before initialization. +- Added a bump-allocation fast path for the current arena chunk. +- Fixed the alignment of external arena allocations, which returned storage + at a misaligned offset for large owned SDS strings. +- Added the `cfl-benchmark-arena-records` benchmark for decoded record + batches. ## 1.0.0 - 2026-07-11 diff --git a/lib/cfl/benchmarks/CMakeLists.txt b/lib/cfl/benchmarks/CMakeLists.txt index f1148df43e8..0a7aefa05c5 100644 --- a/lib/cfl/benchmarks/CMakeLists.txt +++ b/lib/cfl/benchmarks/CMakeLists.txt @@ -3,3 +3,6 @@ target_link_libraries(cfl-benchmark-variant-arena cfl-static) add_executable(cfl-benchmark-variant-mutable variant_mutable.c) target_link_libraries(cfl-benchmark-variant-mutable cfl-static) + +add_executable(cfl-benchmark-arena-records records.c) +target_link_libraries(cfl-benchmark-arena-records cfl-static) diff --git a/lib/cfl/benchmarks/README.md b/lib/cfl/benchmarks/README.md index d4cb58f81fa..3697ad89293 100644 --- a/lib/cfl/benchmarks/README.md +++ b/lib/cfl/benchmarks/README.md @@ -88,3 +88,20 @@ document is still alive. Compare them with `arena_initial_used` and `arena_used`. `max_rss_kb` is process-level high-water RSS and can remain the same for small workloads because it is page-granular and includes executable, library, and allocator-retained pages. + +## Decoded record batches + +The records benchmark builds batches of small log records, each with short +keys, a nested map, an array, and string, integer, and double values, as +produced by decoding a buffer of telemetry records. Arena batches are reset +between iterations; heap batches destroy every record: + +```sh +build-bench/benchmarks/cfl-benchmark-arena-records arena 512 3000 +build-bench/benchmarks/cfl-benchmark-arena-records heap 512 3000 +``` + +Optional arguments select `referenced` or `owned` values, enable record edits +(`1` removes, inserts, and renames keys), and set the arena chunk size. The +tool reports nanoseconds per record and, for arenas, used and reserved bytes +per record. diff --git a/lib/cfl/benchmarks/records.c b/lib/cfl/benchmarks/records.c new file mode 100644 index 00000000000..4c2cd854728 --- /dev/null +++ b/lib/cfl/benchmarks/records.c @@ -0,0 +1,195 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* + * Decoded telemetry batch benchmark: every batch holds many small records + * with short keys, a nested map and an array, as produced by decoding a + * buffer of log records. Values can reference the source buffer or be owned + * copies, and records can be edited after construction. + */ + +#include +#include +#include +#include + +#if !defined(CFL_SYSTEM_WINDOWS) +#include +#endif + +#include + +struct batch_options { + size_t records; + int referenced; + int mutate; +}; + +static int insert_string(struct cfl_kvlist *list, char *key, char *value, + int referenced) +{ + return cfl_kvlist_insert_string_s(list, key, strlen(key), value, + strlen(value), referenced); +} + +static struct cfl_variant *build_record(struct cfl_arena *arena, size_t index, + struct batch_options *options) +{ + char message[64]; + struct cfl_kvlist *record; + struct cfl_kvlist *http; + struct cfl_array *tags; + struct cfl_variant *root; + + snprintf(message, sizeof(message), "request %zu completed", index); + + record = cfl_kvlist_create_in(arena); + http = cfl_kvlist_create_in(arena); + tags = cfl_array_create_in(arena, 2); + if (record == NULL || http == NULL || tags == NULL) { + return NULL; + } + + /* referenced values must outlive the record, keep them static */ + if (insert_string(record, "msg", options->referenced ? + "request completed" : message, + options->referenced) != 0 || + insert_string(record, "level", "info", options->referenced) != 0 || + insert_string(record, "user", "u-1234", options->referenced) != 0 || + insert_string(record, "password", "secret", options->referenced) != 0 || + insert_string(http, "path", "/api/v1/items", options->referenced) != 0 || + insert_string(http, "method", "GET", options->referenced) != 0 || + cfl_kvlist_insert_int64(http, "status", 200) != 0 || + cfl_kvlist_insert_kvlist(record, "http", http) != 0 || + cfl_kvlist_insert_double(record, "duration_ms", 12.5) != 0 || + cfl_array_append_string_s(tags, "edge", 4, options->referenced) != 0 || + cfl_array_append_string_s(tags, "eu-west-1", 9, + options->referenced) != 0 || + cfl_kvlist_insert_array(record, "tags", tags) != 0) { + return NULL; + } + + if (options->mutate) { + if (cfl_kvlist_remove(record, "password") != CFL_TRUE || + cfl_kvlist_insert_int64(record, "x", 1) != 0 || + cfl_kvlist_rename_s(record, "user", 4, "user.id", 7) != 0) { + return NULL; + } + } + + root = cfl_variant_create_from_kvlist_in(arena, record); + return root; +} + +static int run_batch(struct cfl_arena *arena, struct cfl_variant **roots, + struct batch_options *options) +{ + size_t index; + int result; + + result = 0; + for (index = 0; index < options->records; index++) { + roots[index] = build_record(arena, index, options); + if (roots[index] == NULL) { + result = -1; + break; + } + } + + /* arena records are released by the reset, heap ones one by one */ + if (arena == NULL) { + for (index = 0; index < options->records; index++) { + if (roots[index] != NULL) { + cfl_variant_destroy(roots[index]); + } + } + } + + return result; +} + +int main(int argc, char **argv) +{ + const char *mode; + size_t batches; + size_t batch; + size_t chunk_size; + size_t reserved; + size_t used; + uint64_t start; + uint64_t elapsed; + struct cfl_arena *arena; + struct cfl_variant **roots; + struct batch_options options; +#if !defined(CFL_SYSTEM_WINDOWS) + struct rusage usage; +#endif + + mode = argc > 1 ? argv[1] : "arena"; + options.records = argc > 2 ? strtoull(argv[2], NULL, 10) : 512; + batches = argc > 3 ? strtoull(argv[3], NULL, 10) : 2000; + options.referenced = argc > 4 ? strcmp(argv[4], "owned") != 0 : CFL_TRUE; + options.mutate = argc > 5 ? atoi(argv[5]) : 0; + chunk_size = argc > 6 ? strtoull(argv[6], NULL, 10) : 16384; + + if ((strcmp(mode, "arena") != 0 && strcmp(mode, "heap") != 0) || + options.records == 0) { + fprintf(stderr, + "usage: %s arena|heap [records] [batches] " + "[referenced|owned] [mutate] [chunk-size]\n", argv[0]); + return EXIT_FAILURE; + } + + roots = calloc(options.records, sizeof(struct cfl_variant *)); + if (roots == NULL) { + return EXIT_FAILURE; + } + + arena = NULL; + if (strcmp(mode, "arena") == 0) { + arena = cfl_arena_create(chunk_size); + if (arena == NULL) { + free(roots); + return EXIT_FAILURE; + } + } + + reserved = 0; + used = 0; + start = cfl_time_now(); + for (batch = 0; batch < batches; batch++) { + if (run_batch(arena, roots, &options) != 0) { + fprintf(stderr, "batch construction failed\n"); + cfl_arena_destroy(arena); + free(roots); + return EXIT_FAILURE; + } + if (arena != NULL) { + reserved = cfl_arena_bytes_reserved(arena); + used = cfl_arena_bytes_used(arena); + cfl_arena_reset(arena); + } + } + elapsed = cfl_time_now() - start; + + printf("mode=%s records=%zu batches=%zu values=%s mutate=%d " + "elapsed_ns=%llu ns_per_record=%.2f", + mode, options.records, batches, + options.referenced ? "referenced" : "owned", options.mutate, + (unsigned long long) elapsed, + (double) elapsed / (double) (batches * options.records)); + if (arena != NULL) { + printf(" arena_reserved=%zu arena_used=%zu used_per_record=%.1f " + "reserved_per_record=%.1f", reserved, used, + (double) used / (double) options.records, + (double) reserved / (double) options.records); + } +#if !defined(CFL_SYSTEM_WINDOWS) + getrusage(RUSAGE_SELF, &usage); + printf(" max_rss_kb=%ld", usage.ru_maxrss); +#endif + putchar('\n'); + + cfl_arena_destroy(arena); + free(roots); + return EXIT_SUCCESS; +} diff --git a/lib/cfl/src/cfl_arena.c b/lib/cfl/src/cfl_arena.c index 62fa165fddd..a3c80e80a94 100644 --- a/lib/cfl/src/cfl_arena.c +++ b/lib/cfl/src/cfl_arena.c @@ -5,11 +5,15 @@ #include #include +#include +#include +#include #include "cfl_arena_internal.h" #define CFL_ARENA_DEFAULT_CHUNK_SIZE 8192 -#define CFL_ARENA_SDS_CLASS_COUNT 6 +#define CFL_ARENA_SDS_SMALL_PAYLOAD 128 +#define CFL_ARENA_SDS_MEDIUM_CLASS_COUNT 10 #define CFL_ARENA_EXTERNAL_CLASS_COUNT 24 #define CFL_ARENA_EXTERNAL_EXACT_CLASS UINT8_MAX @@ -24,6 +28,72 @@ struct cfl_arena_alignment_probe { union cfl_arena_max_align value; }; +/* + * CFL objects only hold pointers, sizes, integers and doubles, so they are + * packed with the strictest of those alignments instead of the raw allocation + * alignment, which can be larger because of long double. + */ +union cfl_arena_object_align { + void *pointer_value; + size_t size_value; + uint64_t uint64_value; + double double_value; +}; + +struct cfl_arena_object_alignment_probe { + char byte; + union cfl_arena_object_align value; +}; + +#define CFL_ARENA_MAX_ALIGNMENT \ + offsetof(struct cfl_arena_alignment_probe, value) +#define CFL_ARENA_OBJECT_ALIGNMENT \ + offsetof(struct cfl_arena_object_alignment_probe, value) + +/* small SDS classes cover payloads below 128 bytes in alignment steps */ +#define CFL_ARENA_SDS_SMALL_CLASS_COUNT \ + (CFL_ARENA_SDS_SMALL_PAYLOAD / CFL_ARENA_OBJECT_ALIGNMENT) +#define CFL_ARENA_SDS_CLASS_COUNT \ + (CFL_ARENA_SDS_SMALL_CLASS_COUNT + CFL_ARENA_SDS_MEDIUM_CLASS_COUNT) + +struct cfl_arena_variant_probe { + char byte; + struct cfl_variant value; +}; + +struct cfl_arena_kvpair_probe { + char byte; + struct cfl_kvpair value; +}; + +struct cfl_arena_kvlist_probe { + char byte; + struct cfl_kvlist value; +}; + +struct cfl_arena_array_probe { + char byte; + struct cfl_array value; +}; + +/* compile time checks: every object type fits the object alignment */ +typedef char cfl_arena_check_object_alignment[ + (CFL_ARENA_MAX_ALIGNMENT % CFL_ARENA_OBJECT_ALIGNMENT == 0 && + (CFL_ARENA_OBJECT_ALIGNMENT & (CFL_ARENA_OBJECT_ALIGNMENT - 1)) == 0 && + CFL_ARENA_SDS_CLASS_COUNT < UINT8_MAX) ? 1 : -1]; +typedef char cfl_arena_check_variant_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_variant_probe, value) == 0) ? 1 : -1]; +typedef char cfl_arena_check_kvpair_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_kvpair_probe, value) == 0) ? 1 : -1]; +typedef char cfl_arena_check_kvlist_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_kvlist_probe, value) == 0) ? 1 : -1]; +typedef char cfl_arena_check_array_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_array_probe, value) == 0) ? 1 : -1]; + struct cfl_arena_chunk { struct cfl_arena_chunk *next; size_t capacity; @@ -37,6 +107,7 @@ struct cfl_arena_external { struct cfl_arena_external *previous; size_t size; uint8_t allocation_class; + union cfl_arena_max_align alignment; unsigned char data[]; }; @@ -276,24 +347,20 @@ void cfl_arena_reset(struct cfl_arena *arena) memset(arena->free_sds, 0, sizeof(arena->free_sds)); } -void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) +/* walks the remaining chunks, then reserves a new one */ +static void *arena_allocate_slow(struct cfl_arena *arena, size_t size, + size_t alignment) { struct cfl_arena_chunk *chunk; - size_t alignment; size_t offset; size_t remainder; size_t capacity; void *result; - if (arena == NULL || size == 0) { - return NULL; - } - - alignment = offsetof(struct cfl_arena_alignment_probe, value); chunk = arena->current; while (chunk != NULL) { offset = chunk->used; - remainder = offset % alignment; + remainder = offset & (alignment - 1); if (remainder != 0) { if (offset > SIZE_MAX - (alignment - remainder)) { return NULL; @@ -346,6 +413,62 @@ void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) return chunk->data; } +/* bump allocation from the current chunk, the common case */ +static inline void *arena_allocate(struct cfl_arena *arena, size_t size, + size_t alignment) +{ + struct cfl_arena_chunk *chunk; + size_t offset; + void *result; + + if (arena == NULL || size == 0) { + return NULL; + } + + chunk = arena->current; + if (chunk != NULL) { + offset = (chunk->used + alignment - 1) & ~(alignment - 1); + if (offset >= chunk->used && offset <= chunk->capacity && + size <= chunk->capacity - offset) { + result = &chunk->data[offset]; + chunk->used = offset + size; + arena->bytes_used += size; + return result; + } + } + + return arena_allocate_slow(arena, size, alignment); +} + +void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) +{ + return arena_allocate(arena, size, CFL_ARENA_MAX_ALIGNMENT); +} + +void *cfl_arena_alloc_object(struct cfl_arena *arena, size_t size) +{ + return arena_allocate(arena, size, CFL_ARENA_OBJECT_ALIGNMENT); +} + +void *cfl_arena_calloc_object(struct cfl_arena *arena, + size_t count, size_t size) +{ + void *result; + size_t total; + + if (count != 0 && size > SIZE_MAX / count) { + return NULL; + } + + total = count * size; + result = arena_allocate(arena, total, CFL_ARENA_OBJECT_ALIGNMENT); + if (result != NULL) { + memset(result, 0, total); + } + + return result; +} + void *cfl_arena_alloc(struct cfl_arena *arena, size_t size) { return cfl_arena_malloc(arena, size); @@ -604,7 +727,7 @@ void cfl_arena_free_external(struct cfl_arena *arena, } static void *arena_reusable_alloc(struct cfl_arena *arena, - void **free_list, size_t size) + void **free_list, size_t size, int zero) { void *result; @@ -616,11 +739,19 @@ static void *arena_reusable_alloc(struct cfl_arena *arena, result = *free_list; *free_list = *((void **) result); arena->bytes_used += size; + } + else { + result = arena_allocate(arena, size, CFL_ARENA_OBJECT_ALIGNMENT); + if (result == NULL) { + return NULL; + } + } + + if (zero) { memset(result, 0, size); - return result; } - return cfl_arena_calloc(arena, 1, size); + return result; } static void arena_reusable_free(struct cfl_arena *arena, @@ -638,7 +769,7 @@ static void arena_reusable_free(struct cfl_arena *arena, void *cfl_arena_alloc_variant(struct cfl_arena *arena, size_t size) { - return arena_reusable_alloc(arena, &arena->free_variants, size); + return arena_reusable_alloc(arena, &arena->free_variants, size, CFL_TRUE); } void cfl_arena_free_variant(struct cfl_arena *arena, @@ -650,7 +781,7 @@ void cfl_arena_free_variant(struct cfl_arena *arena, void *cfl_arena_alloc_kvpair(struct cfl_arena *arena, size_t size) { - return arena_reusable_alloc(arena, &arena->free_kvpairs, size); + return arena_reusable_alloc(arena, &arena->free_kvpairs, size, CFL_TRUE); } void cfl_arena_free_kvpair(struct cfl_arena *arena, @@ -659,32 +790,80 @@ void cfl_arena_free_kvpair(struct cfl_arena *arena, arena_reusable_free(arena, &arena->free_kvpairs, pointer, size); } +/* + * Owned SDS strings are recycled through size classes. Small classes grow in + * object alignment steps of the complete allocation up to about 128 bytes of + * payload, so short strings such as map keys waste less than one alignment + * unit. Medium classes grow by roughly 25 percent up to the largest pooled + * payload. + */ +static int arena_sds_class(size_t payload_size, size_t overhead_size, + size_t *index, size_t *payload_capacity) +{ + static const size_t medium_payloads[CFL_ARENA_SDS_MEDIUM_CLASS_COUNT] = { + 160, 200, 248, 312, 392, 488, 608, 760, 952, CFL_ARENA_SDS_MAX_PAYLOAD + }; + size_t alignment; + size_t smallest; + size_t total; + size_t class_index; + + alignment = CFL_ARENA_OBJECT_ALIGNMENT; + if (payload_size > CFL_ARENA_SDS_MAX_PAYLOAD || + overhead_size > SIZE_MAX - CFL_ARENA_SDS_MAX_PAYLOAD - 1 - alignment) { + return -1; + } + + /* smallest allocation holds the overhead, the terminator and one byte */ + smallest = (overhead_size + 2 + alignment - 1) & ~(alignment - 1); + total = (overhead_size + payload_size + 1 + alignment - 1) & + ~(alignment - 1); + if (total < smallest) { + total = smallest; + } + + if (total <= smallest + (CFL_ARENA_SDS_SMALL_CLASS_COUNT - 1) * alignment) { + *index = (total - smallest) / alignment; + *payload_capacity = total - overhead_size - 1; + return 0; + } + + for (class_index = 0; class_index < CFL_ARENA_SDS_MEDIUM_CLASS_COUNT; + class_index++) { + total = (overhead_size + medium_payloads[class_index] + 1 + + alignment - 1) & ~(alignment - 1); + if (payload_size <= total - overhead_size - 1) { + *index = CFL_ARENA_SDS_SMALL_CLASS_COUNT + class_index; + *payload_capacity = total - overhead_size - 1; + return 0; + } + } + + return -1; +} + void *cfl_arena_alloc_sds(struct cfl_arena *arena, size_t payload_size, size_t overhead_size, uint8_t *allocation_class, size_t *payload_capacity) { - static const size_t class_sizes[CFL_ARENA_SDS_CLASS_COUNT] = { - 32, 64, 128, 256, 512, 1024 - }; size_t index; if (arena == NULL || allocation_class == NULL || payload_capacity == NULL) { return NULL; } - for (index = 0; index < CFL_ARENA_SDS_CLASS_COUNT; index++) { - if (payload_size <= class_sizes[index]) { - *allocation_class = (uint8_t) (index + 1); - *payload_capacity = class_sizes[index]; - return arena_reusable_alloc(arena, &arena->free_sds[index], - overhead_size + class_sizes[index] + 1); - } + if (arena_sds_class(payload_size, overhead_size, &index, + payload_capacity) != 0) { + *allocation_class = 0; + *payload_capacity = payload_size; + return NULL; } - *allocation_class = 0; - *payload_capacity = payload_size; - return NULL; + /* the SDS layer initializes the headers and the payload it uses */ + *allocation_class = (uint8_t) (index + 1); + return arena_reusable_alloc(arena, &arena->free_sds[index], + overhead_size + *payload_capacity + 1, CFL_FALSE); } void cfl_arena_free_sds(struct cfl_arena *arena, diff --git a/lib/cfl/src/cfl_arena_internal.h b/lib/cfl/src/cfl_arena_internal.h index e41fe3a943b..5959a17ece1 100644 --- a/lib/cfl/src/cfl_arena_internal.h +++ b/lib/cfl/src/cfl_arena_internal.h @@ -2,10 +2,23 @@ #define CFL_ARENA_INTERNAL_H #include +#include #include +/* largest owned SDS payload recycled through arena size classes */ +#define CFL_ARENA_SDS_MAX_PAYLOAD 1024 + void *cfl_arena_alloc(struct cfl_arena *arena, size_t size); + +/* + * CFL object allocations: aligned for the pointers, sizes, integers and + * doubles held by variants, kvpairs, kvlists, arrays and SDS headers, which + * can be less strict than the public raw allocation alignment. + */ +void *cfl_arena_alloc_object(struct cfl_arena *arena, size_t size); +void *cfl_arena_calloc_object(struct cfl_arena *arena, + size_t count, size_t size); void *cfl_arena_alloc_external(struct cfl_arena *arena, size_t size); void cfl_arena_free_external(struct cfl_arena *arena, diff --git a/lib/cfl/src/cfl_array.c b/lib/cfl/src/cfl_array.c index bac55ab67a9..d7b0a99c271 100644 --- a/lib/cfl/src/cfl_array.c +++ b/lib/cfl/src/cfl_array.c @@ -49,7 +49,7 @@ struct cfl_array *cfl_array_create_in(struct cfl_arena *arena, array = malloc(sizeof(struct cfl_array)); } else { - array = cfl_arena_malloc(arena, sizeof(struct cfl_array)); + array = cfl_arena_alloc_object(arena, sizeof(struct cfl_array)); } if (array == NULL) { cfl_errno(); @@ -64,8 +64,8 @@ struct cfl_array *cfl_array_create_in(struct cfl_arena *arena, array->entries = calloc(alloc_count, sizeof(void *)); } else { - array->entries = cfl_arena_calloc(arena, alloc_count, - sizeof(void *)); + array->entries = cfl_arena_calloc_object(arena, alloc_count, + sizeof(void *)); } if (array->entries == NULL) { cfl_errno(); @@ -222,7 +222,7 @@ int cfl_array_append(struct cfl_array *array, tmp = realloc(array->entries, new_size); } else { - tmp = cfl_arena_malloc(array->arena, new_size); + tmp = cfl_arena_alloc_object(array->arena, new_size); if (tmp != NULL) { memcpy(tmp, array->entries, array->entry_count * sizeof(void *)); diff --git a/lib/cfl/src/cfl_kvlist.c b/lib/cfl/src/cfl_kvlist.c index 204ab7744ea..3993e48381a 100644 --- a/lib/cfl/src/cfl_kvlist.c +++ b/lib/cfl/src/cfl_kvlist.c @@ -99,7 +99,7 @@ struct cfl_kvlist *cfl_kvlist_create_in(struct cfl_arena *arena) list = malloc(sizeof(struct cfl_kvlist)); } else { - list = cfl_arena_malloc(arena, sizeof(struct cfl_kvlist)); + list = cfl_arena_alloc_object(arena, sizeof(struct cfl_kvlist)); } if (list == NULL) { cfl_report_runtime_error(); diff --git a/lib/cfl/src/cfl_sds.c b/lib/cfl/src/cfl_sds.c index afe89c2a937..13e39234259 100644 --- a/lib/cfl/src/cfl_sds.c +++ b/lib/cfl/src/cfl_sds.c @@ -82,8 +82,9 @@ static cfl_sds_t sds_alloc(struct cfl_arena *arena, size_t size) if (arena == NULL) { buf = malloc(allocation_size); } - else if (size <= 1024) { - external_threshold = cfl_arena_large_object_threshold(arena); + else if (size <= CFL_ARENA_SDS_MAX_PAYLOAD) { + /* pooled strings never use external storage */ + external_threshold = SIZE_MAX; buf = cfl_arena_alloc_sds(arena, size, CFL_SDS_HEADER_SIZE + sizeof(struct cfl_sds_arena_header), @@ -99,7 +100,7 @@ static cfl_sds_t sds_alloc(struct cfl_arena *arena, size_t size) buf = cfl_arena_alloc_external(arena, allocation_size); } else { - buf = cfl_arena_malloc(arena, allocation_size); + buf = cfl_arena_alloc_object(arena, allocation_size); } } if (!buf) { diff --git a/lib/cfl/tests/arena.c b/lib/cfl/tests/arena.c index 494fa635088..77e4969567b 100644 --- a/lib/cfl/tests/arena.c +++ b/lib/cfl/tests/arena.c @@ -17,6 +17,56 @@ struct test_alignment_probe { union test_max_align value; }; +/* CFL objects hold pointers, sizes, integers and doubles */ +union test_object_align { + void *pointer_value; + size_t size_value; + uint64_t uint64_value; + double double_value; +}; + +struct test_object_alignment_probe { + char byte; + union test_object_align value; +}; + +struct test_variant_probe { + char byte; + struct cfl_variant value; +}; + +struct test_kvpair_probe { + char byte; + struct cfl_kvpair value; +}; + +struct test_kvlist_probe { + char byte; + struct cfl_kvlist value; +}; + +struct test_array_probe { + char byte; + struct cfl_array value; +}; + +#define TEST_MAX_ALIGNMENT offsetof(struct test_alignment_probe, value) +#define TEST_OBJECT_ALIGNMENT offsetof(struct test_object_alignment_probe, value) + +/* owned SDS payloads up to this size are recycled through size classes */ +#define TEST_SDS_POOLED_PAYLOAD 1024 + +static int is_aligned(const void *pointer, size_t alignment) +{ + return ((uintptr_t) pointer % alignment) == 0; +} + +static uint32_t test_random(uint32_t *state) +{ + *state = (*state * 1103515245u) + 12345u; + return (*state >> 8) & 0xffffff; +} + struct test_allocator_context { size_t allocation_count; size_t free_count; @@ -344,6 +394,7 @@ static void reclaim_large_string_on_remove(void) value = cfl_kvlist_fetch(list, "body"); TEST_CHECK(value != NULL); first_buffer = value->data.as_string; + TEST_CHECK(is_aligned(CFL_SDS_HEADER(first_buffer), TEST_OBJECT_ALIGNMENT)); before_remove = cfl_arena_bytes_used(arena); reserved_before_remove = cfl_arena_bytes_reserved(arena); TEST_CHECK(cfl_kvlist_remove(list, "body") == CFL_TRUE); @@ -428,13 +479,15 @@ static void create_like_and_rename(void) static void reuse_sds_size_classes(void) { - static const size_t sizes[] = {1, 32, 33, 64, 65, 128, - 129, 256, 257, 512, 513, 1024}; + static const size_t sizes[] = {0, 1, 7, 8, 15, 16, 31, 32, 33, 64, 65, + 127, 128, 129, 256, 257, 512, 513, 1023, + 1024}; struct cfl_arena *arena; cfl_sds_t first; cfl_sds_t second; cfl_sds_t grown; char payload[1024]; + size_t capacity; size_t index; memset(payload, 's', sizeof(payload)); @@ -444,28 +497,565 @@ static void reuse_sds_size_classes(void) for (index = 0; index < sizeof(sizes) / sizeof(sizes[0]); index++) { first = cfl_sds_create_len_in(arena, payload, (int) sizes[index]); TEST_CHECK(first != NULL); + TEST_CHECK(cfl_sds_alloc(first) >= sizes[index]); cfl_sds_destroy(first); second = cfl_sds_create_len_in(arena, payload, (int) sizes[index]); TEST_CHECK(second == first); cfl_sds_destroy(second); } + /* growing past the capacity moves to the next class, keeping the data */ first = cfl_sds_create_len_in(arena, payload, 31); TEST_CHECK(first != NULL); - TEST_CHECK(cfl_sds_alloc(first) == 32); - grown = cfl_sds_increase(first, 1); + capacity = cfl_sds_alloc(first); + TEST_CHECK(capacity >= 31); + grown = cfl_sds_increase(first, capacity - 31 + 1); TEST_CHECK(grown != NULL); - TEST_CHECK(cfl_sds_alloc(grown) == 64); + TEST_CHECK(grown != first); + TEST_CHECK(cfl_sds_alloc(grown) > capacity); TEST_CHECK(cfl_sds_len(grown) == 31); TEST_CHECK(memcmp(grown, payload, 31) == 0); + TEST_CHECK(grown[31] == '\0'); - second = cfl_sds_create_len_in(arena, payload, 32); + /* the released slot serves the next string of the same class */ + second = cfl_sds_create_len_in(arena, payload, 31); TEST_CHECK(second == first); cfl_sds_destroy(second); cfl_sds_destroy(grown); cfl_arena_destroy(arena); } +static void sds_class_boundaries(void) +{ + struct cfl_arena *arena; + cfl_sds_t value; + cfl_sds_t again; + char payload[TEST_SDS_POOLED_PAYLOAD + 64]; + size_t previous_capacity; + size_t capacity; + size_t used; + size_t size; + + memset(payload, 'b', sizeof(payload)); + arena = cfl_arena_create(4096); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + previous_capacity = 0; + for (size = 0; size <= TEST_SDS_POOLED_PAYLOAD + 32; size++) { + used = cfl_arena_bytes_used(arena); + value = cfl_sds_create_len_in(arena, payload, (int) size); + TEST_CHECK(value != NULL); + if (value == NULL) { + break; + } + + capacity = cfl_sds_alloc(value); + TEST_CHECK(cfl_sds_len(value) == size); + TEST_CHECK(capacity >= size); + TEST_CHECK(cfl_sds_avail(value) == capacity - size); + TEST_CHECK(value[size] == '\0'); + TEST_CHECK(memcmp(value, payload, size) == 0); + TEST_CHECK(is_aligned(CFL_SDS_HEADER(value), TEST_OBJECT_ALIGNMENT)); + + if (size <= TEST_SDS_POOLED_PAYLOAD) { + /* class capacities never shrink as the payload grows */ + TEST_CHECK(capacity >= previous_capacity); + previous_capacity = capacity; + + /* short strings are sized within one alignment unit */ + if (size < 128) { + TEST_CHECK(capacity - size < TEST_OBJECT_ALIGNMENT); + } + else { + TEST_CHECK(capacity - size <= size / 4 + TEST_OBJECT_ALIGNMENT); + } + } + else { + TEST_CHECK(capacity == size); + } + + cfl_sds_destroy(value); + + /* larger strings below the external threshold live until reset */ + if (size <= TEST_SDS_POOLED_PAYLOAD) { + TEST_CHECK(cfl_arena_bytes_used(arena) == used); + again = cfl_sds_create_len_in(arena, payload, (int) size); + TEST_CHECK(again == value); + cfl_sds_destroy(again); + } + } + + cfl_arena_destroy(arena); +} + +static void sds_capacity_is_writable(void) +{ + struct cfl_arena *arena; + cfl_sds_t values[256]; + char payload[2048]; + size_t capacity; + size_t index; + size_t offset; + int intact; + + memset(payload, 'p', sizeof(payload)); + arena = cfl_arena_create(1024); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* fill every string to its reported capacity, neighbours must survive */ + for (index = 0; index < 256; index++) { + values[index] = cfl_sds_create_len_in(arena, payload, + (int) ((index * 37) % 1500)); + TEST_CHECK(values[index] != NULL); + if (values[index] == NULL) { + cfl_arena_destroy(arena); + return; + } + capacity = cfl_sds_alloc(values[index]); + memset(values[index], (int) ('A' + (index % 26)), capacity); + cfl_sds_set_len(values[index], capacity); + TEST_CHECK(values[index][capacity] == '\0'); + } + + for (index = 0; index < 256; index++) { + capacity = cfl_sds_alloc(values[index]); + TEST_CHECK(cfl_sds_len(values[index]) == capacity); + intact = 1; + for (offset = 0; offset < capacity; offset++) { + if (values[index][offset] != (char) ('A' + (index % 26))) { + intact = 0; + break; + } + } + TEST_CHECK(intact == 1); + TEST_CHECK(values[index][capacity] == '\0'); + TEST_MSG("string %zu with capacity %zu", index, capacity); + } + + for (index = 0; index < 256; index += 2) { + cfl_sds_destroy(values[index]); + } + for (index = 1; index < 256; index += 2) { + TEST_CHECK(values[index][0] == (char) ('A' + (index % 26))); + cfl_sds_destroy(values[index]); + } + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + + cfl_arena_destroy(arena); +} + +static void sds_grow_across_classes(void) +{ + struct cfl_arena *arena; + cfl_sds_t value; + cfl_sds_t result; + char expected[3001]; + size_t index; + + arena = cfl_arena_create_ex(2048, 1500); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* small classes, medium classes, chunk storage and external storage */ + value = cfl_sds_create_len_in(arena, "", 0); + TEST_CHECK(value != NULL); + for (index = 0; index < 3000 && value != NULL; index++) { + expected[index] = (char) ('a' + (index % 26)); + result = cfl_sds_cat(value, &expected[index], 1); + TEST_CHECK(result != NULL); + if (result == NULL) { + break; + } + value = result; + TEST_CHECK(is_aligned(CFL_SDS_HEADER(value), TEST_OBJECT_ALIGNMENT)); + TEST_CHECK(cfl_sds_len(value) == index + 1); + TEST_CHECK(cfl_sds_alloc(value) >= index + 1); + TEST_CHECK(value[index + 1] == '\0'); + } + expected[3000] = '\0'; + TEST_CHECK(value != NULL && strcmp(value, expected) == 0); + + /* self append and printf keep working on arena storage */ + result = cfl_sds_cat(value, value, 10); + TEST_CHECK(result != NULL); + if (result != NULL) { + value = result; + TEST_CHECK(cfl_sds_len(value) == 3010); + TEST_CHECK(memcmp(value + 3000, expected, 10) == 0); + } + result = cfl_sds_printf(&value, "-%d", 42); + TEST_CHECK(result != NULL); + TEST_CHECK(cfl_sds_len(value) == 3013); + TEST_CHECK(strcmp(value + 3010, "-42") == 0); + + cfl_sds_destroy(value); + cfl_arena_destroy(arena); +} + +static void object_alignment(void) +{ + struct cfl_arena *arena; + struct cfl_variant *variant; + struct cfl_kvlist *list; + struct cfl_kvpair *pair; + struct cfl_array *array; + struct cfl_variant *entry; + struct cfl_list *head; + char key[8]; + size_t index; + + arena = cfl_arena_create(512); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + list = cfl_kvlist_create_in(arena); + array = cfl_array_create_in(arena, 1); + TEST_CHECK(list != NULL && array != NULL); + if (list == NULL || array == NULL) { + cfl_arena_destroy(arena); + return; + } + TEST_CHECK(cfl_array_resizable(array, CFL_TRUE) == 0); + + /* odd sized keys and payloads shift the bump pointer between objects */ + for (index = 0; index < 64; index++) { + snprintf(key, sizeof(key), "k%zu", index); + TEST_CHECK(cfl_kvlist_insert_string_s(list, key, strlen(key), "abc", + index % 4, CFL_FALSE) == 0); + TEST_CHECK(cfl_kvlist_insert_double(list, "", (double) index) == 0); + TEST_CHECK(cfl_array_append_uint64(array, index) == 0); + + variant = cfl_variant_create_from_int64_in(arena, (int64_t) index); + TEST_CHECK(variant != NULL); + TEST_CHECK(is_aligned(variant, TEST_OBJECT_ALIGNMENT)); + TEST_CHECK(is_aligned(variant, + offsetof(struct test_variant_probe, value))); + cfl_variant_destroy(variant); + } + + TEST_CHECK(is_aligned(list, offsetof(struct test_kvlist_probe, value))); + TEST_CHECK(is_aligned(array, offsetof(struct test_array_probe, value))); + TEST_CHECK(is_aligned(array->entries, sizeof(void *))); + + cfl_list_foreach(head, &list->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + TEST_CHECK(is_aligned(pair, offsetof(struct test_kvpair_probe, value))); + TEST_CHECK(is_aligned(pair->val, + offsetof(struct test_variant_probe, value))); + TEST_CHECK(is_aligned(CFL_SDS_HEADER(pair->key), + TEST_OBJECT_ALIGNMENT)); + } + + for (index = 0; index < cfl_array_size(array); index++) { + entry = cfl_array_fetch_by_index(array, index); + TEST_CHECK(entry != NULL); + TEST_CHECK(is_aligned(entry, offsetof(struct test_variant_probe, value))); + TEST_CHECK(entry->data.as_uint64 == index); + } + + cfl_kvlist_destroy(list); + cfl_array_destroy(array); + cfl_arena_destroy(arena); +} + +static void raw_alignment_after_objects(void) +{ + struct cfl_arena *arena; + struct cfl_variant *variant; + cfl_sds_t key; + void *pointer; + size_t index; + + arena = cfl_arena_create(256); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* raw allocations keep the public alignment between packed objects */ + for (index = 0; index < 64; index++) { + variant = cfl_variant_create_from_bool_in(arena, CFL_TRUE); + TEST_CHECK(variant != NULL); + key = cfl_sds_create_len_in(arena, "key", (int) (index % 5)); + TEST_CHECK(key != NULL); + + pointer = cfl_arena_malloc(arena, 1 + (index % 3)); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + pointer = cfl_arena_calloc(arena, 3, 1); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + pointer = cfl_arena_memdup(arena, "abc", 3); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + pointer = cfl_arena_strndup(arena, "abc", 2); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + } + + cfl_arena_destroy(arena); +} + +static void compact_short_keys(void) +{ + struct cfl_arena *arena; + struct cfl_kvlist *record; + struct cfl_variant *root; + static char *keys[] = {"msg", "user", "path", "status", "level", "ts"}; + size_t record_count; + size_t index; + size_t key_index; + size_t limit; + size_t reserved; + size_t object; + + record_count = 256; + arena = cfl_arena_create(16384); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* worst case: every object padded by one alignment unit */ + object = TEST_OBJECT_ALIGNMENT; + limit = sizeof(struct cfl_kvlist) + object + sizeof(struct cfl_variant) + object; + for (key_index = 0; key_index < 6; key_index++) { + limit += sizeof(struct cfl_kvpair) + object + + sizeof(struct cfl_variant) + object + + 2 * sizeof(void *) + CFL_SDS_HEADER_SIZE + + strlen(keys[key_index]) + 1 + object; + } + + for (index = 0; index < record_count; index++) { + record = cfl_kvlist_create_in(arena); + TEST_CHECK(record != NULL); + if (record == NULL) { + break; + } + for (key_index = 0; key_index < 6; key_index++) { + TEST_CHECK(cfl_kvlist_insert_int64(record, keys[key_index], + (int64_t) index) == 0); + } + root = cfl_variant_create_from_kvlist_in(arena, record); + TEST_CHECK(root != NULL); + } + + /* allow one partially used chunk plus chunk headers */ + reserved = cfl_arena_bytes_reserved(arena); + TEST_CHECK(reserved <= record_count * limit + 16384 + 256); + TEST_MSG("reserved %zu limit %zu", reserved, record_count * limit + 16384 + 256); + + cfl_arena_destroy(arena); +} + +static void reset_reuses_storage_without_aliasing(void) +{ + struct cfl_arena *arena; + cfl_sds_t first; + cfl_sds_t second; + struct cfl_variant *variant; + struct cfl_variant *other; + size_t reserved; + int round; + + arena = cfl_arena_create(1024); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + for (round = 0; round < 8; round++) { + /* released slots must not survive the reset in the free lists */ + first = cfl_sds_create_len_in(arena, "abc", 3); + variant = cfl_variant_create_from_int64_in(arena, round); + TEST_CHECK(first != NULL && variant != NULL); + cfl_sds_destroy(first); + cfl_variant_destroy(variant); + reserved = cfl_arena_bytes_reserved(arena); + cfl_arena_reset(arena); + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + TEST_CHECK(cfl_arena_bytes_reserved(arena) == reserved); + + first = cfl_sds_create_len_in(arena, "abc", 3); + second = cfl_sds_create_len_in(arena, "xyz", 3); + variant = cfl_variant_create_from_int64_in(arena, 1); + other = cfl_variant_create_from_int64_in(arena, 2); + TEST_CHECK(first != NULL && second != NULL); + TEST_CHECK(variant != NULL && other != NULL); + TEST_CHECK(first != second && variant != other); + TEST_CHECK(strcmp(first, "abc") == 0 && strcmp(second, "xyz") == 0); + TEST_CHECK(variant->data.as_int64 == 1 && other->data.as_int64 == 2); + cfl_arena_reset(arena); + } + + cfl_arena_destroy(arena); +} + +static void heap_storage_unchanged(void) +{ + cfl_sds_t value; + struct cfl_kvlist *list; + struct cfl_kvpair *pair; + + /* heap strings keep their exact capacities */ + value = cfl_sds_create_len("abc", 3); + TEST_CHECK(value != NULL); + TEST_CHECK(cfl_sds_alloc(value) == 3); + cfl_sds_destroy(value); + + value = cfl_sds_create_size(100); + TEST_CHECK(value != NULL); + TEST_CHECK(cfl_sds_alloc(value) == 100); + TEST_CHECK(cfl_sds_len(value) == 0); + cfl_sds_destroy(value); + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + if (list == NULL) { + return; + } + TEST_CHECK(cfl_kvlist_insert_int64(list, "status", 200) == 0); + pair = cfl_list_entry_first(&list->list, struct cfl_kvpair, _head); + TEST_CHECK(pair->arena == NULL); + TEST_CHECK(cfl_sds_alloc(pair->key) == 6); + cfl_kvlist_destroy(list); +} + +#define TEST_MUTATION_KEYS 48 + +static void randomized_kvlist_mutation(void) +{ + struct cfl_arena *arena; + struct cfl_kvlist *list; + struct cfl_variant *value; + char *payload; + char keys[TEST_MUTATION_KEYS][224]; + char names[TEST_MUTATION_KEYS][224]; + size_t lengths[TEST_MUTATION_KEYS]; + int present[TEST_MUTATION_KEYS]; + uint32_t state; + size_t key_length; + size_t index; + size_t slot; + size_t count; + int operation; + int round; + int expected; + + payload = malloc(4096); + TEST_CHECK(payload != NULL); + if (payload == NULL) { + return; + } + for (index = 0; index < 4096; index++) { + payload[index] = (char) ('a' + (index % 23)); + } + + /* small chunks and a low external threshold stress every storage path */ + arena = cfl_arena_create_ex(1024, 2048); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + free(payload); + return; + } + + state = 2026; + for (round = 0; round < 4; round++) { + list = cfl_kvlist_create_in(arena); + TEST_CHECK(list != NULL); + if (list == NULL) { + break; + } + + for (slot = 0; slot < TEST_MUTATION_KEYS; slot++) { + key_length = 1 + test_random(&state) % 200; + memset(keys[slot], 'k', key_length); + snprintf(keys[slot], 8, "%02zu", slot); + keys[slot][2] = 'k'; + keys[slot][key_length] = '\0'; + strcpy(names[slot], keys[slot]); + present[slot] = 0; + lengths[slot] = 0; + } + + for (index = 0; index < 4000; index++) { + slot = test_random(&state) % TEST_MUTATION_KEYS; + operation = (int) (test_random(&state) % 4); + + if (operation == 0 && !present[slot]) { + lengths[slot] = test_random(&state) % 3000; + TEST_CHECK(cfl_kvlist_insert_string_s(list, names[slot], + strlen(names[slot]), + payload, lengths[slot], + CFL_FALSE) == 0); + present[slot] = 1; + } + else if (operation == 1 && present[slot]) { + TEST_CHECK(cfl_kvlist_remove(list, names[slot]) == CFL_TRUE); + present[slot] = 0; + } + else if (operation == 2 && present[slot]) { + /* rename between the short and the long spelling */ + if (strcmp(names[slot], keys[slot]) == 0) { + snprintf(names[slot], 8, "%02zu", slot); + } + else { + strcpy(names[slot], keys[slot]); + } + TEST_CHECK(cfl_kvlist_rename_s(list, keys[slot], strlen(keys[slot]), + names[slot], strlen(names[slot])) == 0 || + strcmp(names[slot], keys[slot]) == 0); + if (strcmp(names[slot], keys[slot]) == 0) { + snprintf(names[slot], 8, "%02zu", slot); + TEST_CHECK(cfl_kvlist_rename_s(list, names[slot], + strlen(names[slot]), + keys[slot], + strlen(keys[slot])) == 0); + strcpy(names[slot], keys[slot]); + } + } + else if (operation == 3 && present[slot]) { + TEST_CHECK(cfl_kvlist_remove(list, names[slot]) == CFL_TRUE); + lengths[slot] = test_random(&state) % 3000; + TEST_CHECK(cfl_kvlist_insert_string_s(list, names[slot], + strlen(names[slot]), + payload, lengths[slot], + CFL_FALSE) == 0); + } + } + + count = 0; + for (slot = 0; slot < TEST_MUTATION_KEYS; slot++) { + value = cfl_kvlist_fetch(list, names[slot]); + expected = present[slot]; + TEST_CHECK((value != NULL) == expected); + if (value == NULL || !expected) { + continue; + } + count++; + TEST_CHECK(value->type == CFL_VARIANT_STRING); + TEST_CHECK(cfl_sds_len(value->data.as_string) == lengths[slot]); + TEST_CHECK(memcmp(value->data.as_string, payload, + lengths[slot]) == 0); + TEST_CHECK(value->data.as_string[lengths[slot]] == '\0'); + } + TEST_CHECK((size_t) cfl_kvlist_count(list) == count); + + cfl_kvlist_destroy(list); + cfl_arena_reset(arena); + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + } + + cfl_arena_destroy(arena); + free(payload); +} + static void bound_external_rounding_and_cache(void) { struct cfl_arena *arena; @@ -523,8 +1113,7 @@ static void reclaim_failed_variant_construction(void) reused = cfl_variant_create_from_int64_in(arena, 1); TEST_CHECK(reused != NULL); - TEST_CHECK(((uintptr_t) reused % - offsetof(struct test_alignment_probe, value)) == 0); + TEST_CHECK(is_aligned(reused, offsetof(struct test_variant_probe, value))); cfl_variant_destroy(reused); cfl_arena_destroy(arena); } @@ -542,6 +1131,15 @@ TEST_LIST = { {"reuse_variant_and_kvpair_slots", reuse_variant_and_kvpair_slots}, {"create_like_and_rename", create_like_and_rename}, {"reuse_sds_size_classes", reuse_sds_size_classes}, + {"sds_class_boundaries", sds_class_boundaries}, + {"sds_capacity_is_writable", sds_capacity_is_writable}, + {"sds_grow_across_classes", sds_grow_across_classes}, + {"object_alignment", object_alignment}, + {"raw_alignment_after_objects", raw_alignment_after_objects}, + {"compact_short_keys", compact_short_keys}, + {"reset_reuses_storage_without_aliasing", reset_reuses_storage_without_aliasing}, + {"heap_storage_unchanged", heap_storage_unchanged}, + {"randomized_kvlist_mutation", randomized_kvlist_mutation}, {"bound_external_rounding_and_cache", bound_external_rounding_and_cache}, {"reclaim_failed_variant_construction", reclaim_failed_variant_construction}, {NULL, NULL} From 3972cce8c1aaf510bc4329d343b3a2fc5a048438 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 20:36:21 -0600 Subject: [PATCH 20/61] mp: add log windows and appending chunk encoding flb_mp_log_window_next() finds group aligned windows of a log buffer by size and record count. flb_mp_chunk_cobj_encode_append() appends a chunk to its encoder output so several chunks can produce one buffer. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_mp.h | 3 + include/fluent-bit/flb_mp_chunk.h | 1 + src/flb_mp.c | 140 ++++++++++++++++++++++++++++-- 3 files changed, 137 insertions(+), 7 deletions(-) diff --git a/include/fluent-bit/flb_mp.h b/include/fluent-bit/flb_mp.h index 831f1666bfa..d778f47cc01 100644 --- a/include/fluent-bit/flb_mp.h +++ b/include/fluent-bit/flb_mp.h @@ -34,6 +34,9 @@ int flb_mp_count(const void *data, size_t bytes); */ int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *records, int *plain); int flb_mp_count_log_records(const void *data, size_t bytes); +int flb_mp_log_window_next(const void *data, size_t bytes, size_t offset, + size_t limit_bytes, int limit_records, + size_t *window_end, int *window_records); int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_size, char **out_buf, size_t *out_size); int flb_mp_normalize_log_buffer_groups(const void *in_buf, size_t in_size, diff --git a/include/fluent-bit/flb_mp_chunk.h b/include/fluent-bit/flb_mp_chunk.h index 86e59e075c6..58d8fc0233d 100644 --- a/include/fluent-bit/flb_mp_chunk.h +++ b/include/fluent-bit/flb_mp_chunk.h @@ -89,6 +89,7 @@ void flb_mp_chunk_cobj_value_pool_enable(struct flb_mp_chunk_cobj *chunk_cobj); struct cfl_arena *flb_mp_chunk_cobj_value_pool_get(struct flb_mp_chunk_cobj *chunk_cobj, size_t estimated_bytes); +int flb_mp_chunk_cobj_encode_append(struct flb_mp_chunk_cobj *chunk_cobj); int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size); int flb_mp_chunk_cobj_count_log_records(struct flb_mp_chunk_cobj *chunk_cobj); int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj); diff --git a/src/flb_mp.c b/src/flb_mp.c index caa33018b9a..53231c60fd0 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -262,6 +262,120 @@ int flb_mp_count_log_records(const void *data, size_t bytes) return count; } +/* + * Find the end of the log window starting at 'offset'. A window closes after + * 'limit_bytes' of serialized events or 'limit_records' log records, but only + * outside a group: every group stays whole within one window, which can exceed + * the limits by up to one group. + */ +int flb_mp_log_window_next(const void *data, size_t bytes, size_t offset, + size_t limit_bytes, int limit_records, + size_t *window_end, int *window_records) +{ + mpack_reader_t reader; + mpack_tag_t tag; + msgpack_object timestamp = {0}; + struct flb_time time; + char extension[8]; + size_t position = 0; + int header; + int records = 0; + int in_group = FLB_FALSE; + int ret = -1; + + if (!data || !window_end || !window_records || offset >= bytes) { + return -1; + } + + mpack_reader_init_data(&reader, (const char *) data + offset, bytes - offset); + + while (mpack_reader_remaining(&reader, NULL) > 0) { + /* every event is a two entries array: [header|timestamp, body] */ + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_array || mpack_tag_array_count(&tag) != 2) { + goto done; + } + + /* the header form is [timestamp, metadata] */ + tag = mpack_read_tag(&reader); + header = mpack_tag_type(&tag) == mpack_type_array; + if (header) { + if (mpack_tag_array_count(&tag) != 2) { + goto done; + } + tag = mpack_read_tag(&reader); + } + + if (mpack_tag_type(&tag) == mpack_type_uint) { + timestamp.type = MSGPACK_OBJECT_POSITIVE_INTEGER; + timestamp.via.u64 = mpack_tag_uint_value(&tag); + } + else if (mpack_tag_type(&tag) == mpack_type_double) { + timestamp.type = MSGPACK_OBJECT_FLOAT; + timestamp.via.f64 = mpack_tag_double_value(&tag); + } + else if (mpack_tag_type(&tag) == mpack_type_float) { + timestamp.type = MSGPACK_OBJECT_FLOAT; + timestamp.via.f64 = mpack_tag_float_value(&tag); + } + else if (mpack_tag_type(&tag) == mpack_type_ext && + mpack_tag_ext_length(&tag) == 8 && mpack_tag_ext_exttype(&tag) == 0) { + mpack_read_bytes(&reader, extension, sizeof(extension)); + mpack_done_ext(&reader); + + timestamp.type = MSGPACK_OBJECT_EXT; + timestamp.via.ext.ptr = extension; + timestamp.via.ext.size = sizeof(extension); + timestamp.via.ext.type = 0; + } + else { + goto done; + } + + if (header) { + mpack_discard(&reader); + mpack_done_array(&reader); + } + + /* body */ + mpack_discard(&reader); + mpack_done_array(&reader); + + if (mpack_reader_error(&reader) != mpack_ok || + flb_log_event_decoder_decode_timestamp(×tamp, &time) != 0) { + goto done; + } + + if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_START) { + in_group = FLB_TRUE; + } + else if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { + in_group = FLB_FALSE; + } + else if (records < INT_MAX) { + records++; + } + + position = (bytes - offset) - mpack_reader_remaining(&reader, NULL); + if (!in_group && (position >= limit_bytes || records >= limit_records)) { + break; + } + } + + if (position > 0) { + *window_end = offset + position; + *window_records = records; + ret = 0; + } + +done: + if (mpack_reader_destroy(&reader) != mpack_ok) { + ret = -1; + } + + return ret; +} + struct flb_mp_record_span { const char *base; size_t length; @@ -1734,7 +1848,11 @@ static int encode_cfl_field(struct cfl_object *obj, msgpack_sbuffer *buffer, return mp_cfl_to_msgpack(obj->variant, buffer, packer); } -int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size) +/* + * Append the chunk records to its log encoder output. A failure resets the + * encoder, including the output of previous appends. + */ +int flb_mp_chunk_cobj_encode_append(struct flb_mp_chunk_cobj *chunk_cobj) { int ret; struct cfl_list *head; @@ -1800,12 +1918,6 @@ int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_bu } msgpack_sbuffer_destroy(&buffer); - - /* set new output buffer */ - *out_buf = chunk_cobj->log_encoder->output_buffer; - *out_size = chunk_cobj->log_encoder->output_length; - - flb_log_event_encoder_claim_internal_buffer_ownership(chunk_cobj->log_encoder); return 0; failure: @@ -1816,6 +1928,20 @@ int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_bu return -1; } +int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size) +{ + if (flb_mp_chunk_cobj_encode_append(chunk_cobj) != 0) { + return -1; + } + + /* set new output buffer */ + *out_buf = chunk_cobj->log_encoder->output_buffer; + *out_size = chunk_cobj->log_encoder->output_length; + + flb_log_event_encoder_claim_internal_buffer_ownership(chunk_cobj->log_encoder); + return 0; +} + int flb_mp_chunk_cobj_count_log_records(struct flb_mp_chunk_cobj *chunk_cobj) { int count; From b7bf5ecc1c3405abdc909ad6cf74f7ef435724da Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 20:36:21 -0600 Subject: [PATCH 21/61] processor: process large log buffers in windows Native processors decode the complete buffer before running, so output processors held every record of a stored chunk decoded at once. When all native units of a run declare FLB_PROCESSOR_FLAG_LOG_WINDOWS, buffers larger than 64 KiB are decoded, processed and encoded in windows of about 64 KiB or 1024 records. Windows never split a group, and malformed buffers keep the single chunk path. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_processor.h | 11 +- src/flb_processor.c | 285 +++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+), 1 deletion(-) diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index 2a3805a9ab0..8b47488442a 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -44,6 +44,15 @@ #define FLB_PROCESSOR_UNIT_NATIVE 0 #define FLB_PROCESSOR_UNIT_FILTER 1 +/* + * Plugin flags + * + * FLB_PROCESSOR_FLAG_LOG_WINDOWS: the logs callback only depends on each + * record and its group, so the processor chain can run it over large buffers + * in group aligned windows. + */ +#define FLB_PROCESSOR_FLAG_LOG_WINDOWS 1 + /* The current values mean the processor stack will * wait for 2 seconds at most in 50 millisecond increments @@ -148,7 +157,7 @@ struct flb_processor { }; struct flb_processor_plugin { - int flags; /* Flags (not available at the moment */ + int flags; /* FLB_PROCESSOR_FLAG_* capabilities */ char *name; /* Processor short name */ char *description; /* Description */ diff --git a/src/flb_processor.c b/src/flb_processor.c index c5f6badcbdf..2937fd50507 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -1314,6 +1314,245 @@ int flb_processor_is_active(struct flb_processor *proc) * context for metrics, a 'CTraces' context for traces or a 'CProfiles' context for * profiles. */ +/* + * Large log buffers, such as complete chunks reaching output processors, are + * processed in windows when every native unit of the run is record local: + * each window is decoded, processed by all the units, encoded and released + * before the next one, so decoded records never hold more than one window. + * Windows only close outside groups, keeping every group in one window. + */ +#define FLB_PROCESSOR_LOG_WINDOW_BYTES 65536 +#define FLB_PROCESSOR_LOG_WINDOW_RECORDS 1024 + +#define LOG_WINDOWS_DONE 0 +#define LOG_WINDOWS_SKIP 1 +#define LOG_WINDOWS_ERROR -1 + +struct log_window_unit { + struct flb_processor_unit *pu; + int in_items; + int out_items; + int error; +}; + +/* last unit of the native run starting at 'head' if windows can be used */ +static struct mk_list *log_window_run_end(struct mk_list *head, struct mk_list *list, + int *units) +{ + struct mk_list *last = NULL; + struct flb_processor_unit *pu; + struct flb_processor_instance *p_ins; + + *units = 0; + for (; head != list; head = head->next) { + pu = mk_list_entry(head, struct flb_processor_unit, _head); + if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { + break; + } + + p_ins = pu->ctx; + if (p_ins->p->cb_process_logs == NULL || + (p_ins->p->flags & FLB_PROCESSOR_FLAG_LOG_WINDOWS) == 0) { + return NULL; + } + + last = head; + (*units)++; + } + + return last; +} + +struct log_window { + size_t end; + int records; +}; + +/* window boundaries for the complete buffer, NULL if one window is enough */ +static struct log_window *log_windows_scan(void *data, size_t data_size, int *count) +{ + int records; + int capacity = 0; + size_t offset = 0; + size_t end; + struct log_window *tmp; + struct log_window *windows = NULL; + + *count = 0; + while (offset < data_size) { + if (flb_mp_log_window_next(data, data_size, offset, + FLB_PROCESSOR_LOG_WINDOW_BYTES, + FLB_PROCESSOR_LOG_WINDOW_RECORDS, + &end, &records) != 0) { + /* the single chunk path reports invalid content as usual */ + flb_free(windows); + return NULL; + } + + if (*count == capacity) { + capacity = capacity ? capacity * 2 : 32; + tmp = flb_realloc(windows, sizeof(struct log_window) * capacity); + if (!tmp) { + flb_errno(); + flb_free(windows); + return NULL; + } + windows = tmp; + } + + windows[*count].end = end; + windows[*count].records = records; + (*count)++; + offset = end; + } + + if (*count < 2) { + flb_free(windows); + return NULL; + } + + return windows; +} + +static int run_log_windows(struct flb_processor *proc, + struct mk_list *first, int unit_count, + const char *tag, size_t tag_len, + void *data, size_t data_size, + void **out_buf, size_t *out_size, + int *out_records, int *out_normalized, + const char *metrics_scope, + const char *metrics_owner, + const char *metrics_signal) +{ + int i; + int w; + int ret; + int result = LOG_WINDOWS_ERROR; + int records; + int total_records = 0; + int normalized = FLB_TRUE; + int window_count; + size_t offset; + struct mk_list *head; + struct log_window *windows; + struct log_window_unit *units; + struct flb_processor_instance *p_ins; + struct flb_processor_instance *owner; + struct flb_mp_chunk_cobj *chunk_cobj = NULL; + + windows = log_windows_scan(data, data_size, &window_count); + if (!windows) { + return LOG_WINDOWS_SKIP; + } + + units = flb_calloc(unit_count, sizeof(struct log_window_unit)); + if (!units) { + flb_errno(); + flb_free(windows); + return LOG_WINDOWS_SKIP; + } + + head = first; + for (i = 0; i < unit_count; i++) { + units[i].pu = mk_list_entry(head, struct flb_processor_unit, _head); + head = head->next; + } + + /* the first unit owns the decoder and the encoder, as in a single chunk */ + owner = units[0].pu->ctx; + + offset = 0; + for (w = 0; w < window_count; w++) { + flb_log_event_decoder_reset(owner->log_decoder, (char *) data + offset, + windows[w].end - offset); + chunk_cobj = flb_mp_chunk_cobj_create(owner->log_encoder, owner->log_decoder); + if (!chunk_cobj) { + goto error; + } + + records = windows[w].records; + for (i = 0; i < unit_count; i++) { + p_ins = units[i].pu->ctx; + + ret = acquire_lock(&units[i].pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + if (ret != FLB_TRUE) { + units[i].error = FLB_TRUE; + goto error; + } + + chunk_cobj->condition = units[i].pu->condition; + ret = p_ins->p->cb_process_logs(p_ins, chunk_cobj, tag, tag_len); + if (ret != FLB_PROCESSOR_SUCCESS) { + units[i].error = FLB_TRUE; + flb_warn("[processor] failed to process chunk"); + } + chunk_cobj->record_pos = NULL; + chunk_cobj->condition = NULL; + + release_lock(&units[i].pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + + units[i].in_items += records; + records = flb_mp_chunk_cobj_count_log_records(chunk_cobj); + units[i].out_items += records; + } + + if (flb_mp_chunk_cobj_normalize_groups(chunk_cobj) != 0) { + normalized = FLB_FALSE; + } + total_records += flb_mp_chunk_cobj_count_log_records(chunk_cobj); + + if (cfl_list_size(&chunk_cobj->records) > 0 && + flb_mp_chunk_cobj_encode_append(chunk_cobj) != 0) { + goto error; + } + + flb_mp_chunk_cobj_destroy(chunk_cobj); + chunk_cobj = NULL; + offset = windows[w].end; + } + + if (total_records == 0 || owner->log_encoder->output_length == 0) { + flb_log_event_encoder_reset(owner->log_encoder); + *out_buf = NULL; + *out_size = 0; + total_records = 0; + } + else { + *out_buf = owner->log_encoder->output_buffer; + *out_size = owner->log_encoder->output_length; + flb_log_event_encoder_claim_internal_buffer_ownership(owner->log_encoder); + } + + *out_records = total_records; + *out_normalized = normalized; + result = LOG_WINDOWS_DONE; + goto exit; + +error: + if (chunk_cobj) { + flb_mp_chunk_cobj_destroy(chunk_cobj); + } + flb_log_event_encoder_reset(owner->log_encoder); + +exit: + flb_log_event_decoder_reset(owner->log_decoder, NULL, 0); + + for (i = 0; i < unit_count; i++) { + processor_metrics_update(proc, units[i].pu, + metrics_scope, metrics_owner, metrics_signal, + units[i].in_items, units[i].out_items, + FLB_TRUE, units[i].error); + } + flb_free(units); + flb_free(windows); + + return result; +} + int flb_processor_run(struct flb_processor *proc, size_t starting_stage, int type, @@ -1352,6 +1591,10 @@ int flb_processor_run_with_count(struct flb_processor *proc, struct flb_filter_instance *f_ins; struct flb_processor_instance *p_ins; struct flb_mp_chunk_cobj *chunk_cobj = NULL; + struct mk_list *run_end; + int run_units; + int window_records; + int window_normalized; const char *metrics_scope; const char *metrics_owner; const char *metrics_signal; @@ -1423,6 +1666,48 @@ int flb_processor_run_with_count(struct flb_processor *proc, item_metrics_available = FLB_TRUE; } + /* large buffers run record local native units in bounded windows */ + if (type == FLB_PROCESSOR_LOGS && chunk_cobj == NULL && + pu->unit_type == FLB_PROCESSOR_UNIT_NATIVE && + cur_size > FLB_PROCESSOR_LOG_WINDOW_BYTES) { + run_end = log_window_run_end(head, list, &run_units); + if (run_end != NULL) { + ret = run_log_windows(proc, head, run_units, tag, tag_len, + cur_buf, cur_size, &tmp_buf, &tmp_size, + &window_records, &window_normalized, + metrics_scope, metrics_owner, metrics_signal); + if (ret == LOG_WINDOWS_ERROR) { + if (cur_buf != data) { + flb_free(cur_buf); + } + return -1; + } + else if (ret == LOG_WINDOWS_DONE) { + if (cur_buf != data) { + flb_free(cur_buf); + } + + if (tmp_buf == NULL) { + *out_buf = NULL; + *out_size = 0; + if (output_records) { + *output_records = 0; + } + return 0; + } + + cur_buf = tmp_buf; + cur_size = tmp_size; + log_buffer_records = window_records; + log_buffer_normalized = window_normalized; + + /* continue after the last unit of the run */ + head = run_end; + continue; + } + } + } + ret = acquire_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); From 377dcee034e26ea473b9b356f2f1bfd99eb2d271 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 20:36:21 -0600 Subject: [PATCH 22/61] processor_fts: enable windowed log processing Signed-off-by: Eduardo Silva --- plugins/processor_fts/README.md | 16 +++++++++++----- plugins/processor_fts/fts.c | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/plugins/processor_fts/README.md b/plugins/processor_fts/README.md index c626559f633..0cbc3fc8aeb 100644 --- a/plugins/processor_fts/README.md +++ b/plugins/processor_fts/README.md @@ -60,7 +60,7 @@ Use one processor entry and one script for each signal. | `instruction_limit` | `1000000` | Maximum Lua instructions per callback and during script load. Minimum `1000`. `0` is allowed only with `jit: true`. | | `jit` | `false` | Enable the LuaJIT compiler. Requires `instruction_limit: 0`. | | `memory_limit` | `32M` | Maximum memory for the policy: the Lua heap plus the typed values it holds. Minimum `262144`. | -| `output_limit` | `64M` | Maximum serialized size of one processed chunk. Minimum `65536`. | +| `output_limit` | `64M` | Maximum serialized size of one processed batch. Minimum `65536`. | Size suffixes are decimal: `64K` is 64000 bytes. @@ -75,8 +75,14 @@ Size suffixes are decimal: `64K` is 64000 bytes. Changes are transactional per item. Edits to the body, metadata, timestamp, resource, and scope are committed or rolled back together. -If a processed chunk exceeds `output_limit`, the whole chunk is treated as -failed: `drop` discards it and `keep` forwards the original chunk. +If a processed batch exceeds `output_limit`, the whole batch is treated as +failed: `drop` discards it and `keep` forwards the original batch. + +A batch is the set of records handed to one callback run. Small chunks are +processed as one batch. Larger buffers, such as complete chunks reaching output +processors, are processed in batches of about 64 KiB or 1024 records, so +decoded records never hold more than one batch in memory. Groups, such as one +OpenTelemetry resource and scope, are never split across batches. For privacy policies such as redaction, keep the default `error_mode: drop`. With `keep`, an item whose policy fails is forwarded unmodified. @@ -316,6 +322,6 @@ message text need an explicit string policy. ## Diagnostics -Each chunk logs at most five policy errors plus a summary count. The processor -error metric counts chunks with handled policy failures, and the drop metrics +Each batch logs at most five policy errors plus a summary count. The processor +error metric counts batches with handled policy failures, and the drop metrics count removed records. diff --git a/plugins/processor_fts/fts.c b/plugins/processor_fts/fts.c index 2393aa8c709..fe62b3a5bf9 100644 --- a/plugins/processor_fts/fts.c +++ b/plugins/processor_fts/fts.c @@ -172,7 +172,7 @@ static struct flb_config_map config_map[] = { { FLB_CONFIG_MAP_SIZE, "output_limit", "64M", 0, FLB_TRUE, offsetof(struct fts_runtime, output_limit), - "Maximum serialized size of a processed chunk." + "Maximum serialized size of a processed batch." }, { FLB_CONFIG_MAP_SIZE, "memory_limit", "32M", @@ -193,5 +193,5 @@ struct flb_processor_plugin processor_fts_plugin = { .cb_process_traces = fts_process_traces, .cb_exit = cb_exit, .config_map = config_map, - .flags = 0 + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS }; From 2e2161e6b8ed362ed5e159c5e7827b23b483b2bf Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 20:36:21 -0600 Subject: [PATCH 23/61] processor_content_modifier: enable windowed log processing Signed-off-by: Eduardo Silva --- plugins/processor_content_modifier/cm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/processor_content_modifier/cm.c b/plugins/processor_content_modifier/cm.c index 22180dbca17..ebc34831b2e 100644 --- a/plugins/processor_content_modifier/cm.c +++ b/plugins/processor_content_modifier/cm.c @@ -167,5 +167,5 @@ struct flb_processor_plugin processor_content_modifier_plugin = { .cb_process_traces = cb_process_traces, .cb_exit = cb_exit, .config_map = config_map, - .flags = 0 + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS }; From 0055b7f0304d1e6dc254fe9a73d8cbc4c795945e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 20:36:21 -0600 Subject: [PATCH 24/61] tests: internal: mp: cover log windows Signed-off-by: Eduardo Silva --- tests/internal/mp.c | 95 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/tests/internal/mp.c b/tests/internal/mp.c index ee556122f99..4aa4461e320 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -809,8 +809,103 @@ static void test_log_scan_matches_decoder(void) flb_log_event_encoder_destroy(encoder); } +static void append_window_test_record(struct flb_log_event_encoder *encoder, int seq) +{ + int ret; + struct flb_time timestamp; + + TEST_ASSERT(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(×tamp, 1700000000 + seq, 0); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(encoder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("seq"), FLB_LOG_EVENT_INT64_VALUE(seq)); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(encoder) == 0); +} + +static void test_log_window_next(void) +{ + struct flb_log_event_encoder *encoder; + size_t offset; + size_t end; + size_t group_start; + size_t group_end; + int records; + int total; + int windows; + int seq; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + + /* 10 plain records, a group of 6 records, 4 plain records */ + for (seq = 0; seq < 10; seq++) { + append_window_test_record(encoder, seq); + } + group_start = encoder->output_length; + TEST_ASSERT(flb_log_event_encoder_group_init(encoder) == 0); + TEST_ASSERT(flb_log_event_encoder_group_header_end(encoder) == 0); + for (; seq < 16; seq++) { + append_window_test_record(encoder, seq); + } + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + group_end = encoder->output_length; + for (; seq < 20; seq++) { + append_window_test_record(encoder, seq); + } + + /* record limited windows cover the buffer without splitting the group */ + offset = 0; + total = 0; + windows = 0; + while (offset < encoder->output_length) { + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, + encoder->output_length, offset, + SIZE_MAX, 4, &end, &records) == 0); + TEST_CHECK(end > offset && end <= encoder->output_length); + TEST_CHECK(!(offset < group_start && end > group_start && end < group_end)); + TEST_CHECK(!(offset > group_start && offset < group_end)); + TEST_CHECK(flb_mp_count_log_records(encoder->output_buffer + offset, + end - offset) == records); + if (end <= group_start || offset >= group_end) { + TEST_CHECK(records <= 4); + } + total += records; + windows++; + offset = end; + } + TEST_CHECK(offset == encoder->output_length); + TEST_CHECK(total == 20); + /* 4 + 4 + (2 plain and the whole group of 6) + 4 */ + TEST_CHECK(windows == 4); + + /* a byte limit of one closes after every event outside the group */ + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + 0, 1, INT_MAX, &end, &records) == 0); + TEST_CHECK(records == 1); + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + group_start, 1, INT_MAX, &end, &records) == 0); + TEST_CHECK(end == group_end && records == 6); + + /* a single window for the whole buffer */ + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + 0, SIZE_MAX, INT_MAX, &end, &records) == 0); + TEST_CHECK(end == encoder->output_length && records == 20); + + /* invalid input and offsets */ + TEST_CHECK(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + encoder->output_length, 1, 1, &end, &records) == -1); + TEST_CHECK(flb_mp_log_window_next(NULL, 10, 0, 1, 1, &end, &records) == -1); + TEST_CHECK(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length - 1, + 0, SIZE_MAX, INT_MAX, &end, &records) == -1); + TEST_CHECK(flb_mp_log_window_next("\x92\x01", 2, 0, 1, 1, &end, &records) == -1); + + flb_log_event_encoder_destroy(encoder); +} + TEST_LIST = { {"log_scan_matches_decoder", test_log_scan_matches_decoder}, + {"log_window_next", test_log_window_next}, {"preflight", test_preflight}, {"preflight_formats", test_preflight_formats}, {"validate_trace_chunk" , test_validate_trace_chunk}, From 62e69a2e9e9bbdb1a6877ba3aeb7e91158ce8e7f Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 20:36:21 -0600 Subject: [PATCH 25/61] tests: internal: processor: cover windowed log processing Signed-off-by: Eduardo Silva --- tests/internal/processor.c | 486 +++++++++++++++++++++++++++++++++++++ 1 file changed, 486 insertions(+) diff --git a/tests/internal/processor.c b/tests/internal/processor.c index 7c6481e5029..fc260fe8cb8 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include @@ -684,11 +686,495 @@ static void processor_metrics_failure_clears_output(void) flb_config_exit(config); } +/* records per group and around the groups of the windowed buffer */ +#define WINDOW_GROUPS 3 +#define WINDOW_GROUP_RECORDS 1500 +#define WINDOW_PLAIN_RECORDS 2000 +#define WINDOW_TOTAL_RECORDS (WINDOW_GROUPS * WINDOW_GROUP_RECORDS + \ + 2 * WINDOW_PLAIN_RECORDS) + +static int window_calls; +static int window_max_records; + +static int append_window_record(struct flb_log_event_encoder *encoder, int seq) +{ + int ret; + struct flb_time ts; + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + flb_time_set(&ts, 1700000000 + seq, 0); + ret = flb_log_event_encoder_set_timestamp(encoder, &ts); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_CSTRING_VALUE("seq"), + FLB_LOG_EVENT_INT64_VALUE(seq), + FLB_LOG_EVENT_CSTRING_VALUE("message"), + FLB_LOG_EVENT_CSTRING_VALUE("windowed processing test record")); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_commit_record(encoder); + } + + return ret == FLB_EVENT_ENCODER_SUCCESS ? 0 : -1; +} + +/* plain records, groups larger than the window record limit, plain records */ +static int create_window_records(char **out_buf, size_t *out_size) +{ + int i; + int g; + int seq = 0; + int ret = 0; + char name[8]; + struct flb_log_event_encoder *encoder; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + for (i = 0; ret == 0 && i < WINDOW_PLAIN_RECORDS; i++) { + ret = append_window_record(encoder, seq++); + } + + for (g = 0; ret == 0 && g < WINDOW_GROUPS; g++) { + snprintf(name, sizeof(name), "g%d", g); + if (flb_log_event_encoder_group_init(encoder) != 0) { + ret = -1; + break; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_CSTRING_VALUE("group"), + FLB_LOG_EVENT_CSTRING_VALUE(name)); + if (ret != FLB_EVENT_ENCODER_SUCCESS || + flb_log_event_encoder_group_header_end(encoder) != 0) { + ret = -1; + break; + } + + for (i = 0; ret == 0 && i < WINDOW_GROUP_RECORDS; i++) { + ret = append_window_record(encoder, seq++); + } + + if (ret == 0 && flb_log_event_encoder_group_end(encoder) != 0) { + ret = -1; + } + } + + for (i = 0; ret == 0 && i < WINDOW_PLAIN_RECORDS; i++) { + ret = append_window_record(encoder, seq++); + } + + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + *out_buf = flb_malloc(encoder->output_length); + if (*out_buf == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + memcpy(*out_buf, encoder->output_buffer, encoder->output_length); + *out_size = encoder->output_length; + flb_log_event_encoder_destroy(encoder); + + return 0; +} + +static int mark_window_records(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + int type; + int records = 0; + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_record *record; + + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + if (flb_log_event_decoder_get_record_type(&record->event, &type) != 0 || + type != FLB_LOG_EVENT_NORMAL) { + continue; + } + records++; + if (cfl_kvlist_insert_int64(record->cobj_record->variant->data.as_kvlist, + "x", 1) != 0) { + return FLB_PROCESSOR_FAILURE; + } + } + + window_calls++; + if (records > window_max_records) { + window_max_records = records; + } + + return FLB_PROCESSOR_SUCCESS; +} + +static int drop_window_records(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + int type; + struct cfl_list *head; + struct cfl_list *tmp; + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_record *record; + + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + } + chunk->record_pos = NULL; + + cfl_list_foreach_safe(head, tmp, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + if (flb_log_event_decoder_get_record_type(&record->event, &type) == 0 && + type == FLB_LOG_EVENT_NORMAL) { + flb_mp_chunk_cobj_record_destroy(chunk, record); + } + } + + window_calls++; + return FLB_PROCESSOR_SUCCESS; +} + +static struct flb_processor_plugin window_mark_plugin = { + .name = "test_window_mark", + .description = "windowed record marker", + .cb_process_logs = mark_window_records, + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS +}; + +static struct flb_processor_plugin single_mark_plugin = { + .name = "test_single_mark", + .description = "single chunk record marker", + .cb_process_logs = mark_window_records, + .flags = 0 +}; + +static struct flb_processor_plugin window_drop_plugin = { + .name = "test_window_drop", + .description = "windowed record dropper", + .cb_process_logs = drop_window_records, + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS +}; + +static struct flb_config *window_config_create(void) +{ + struct flb_config *config; + + flb_init_env(); + config = flb_config_init(); + if (config == NULL) { + return NULL; + } + mk_list_add(&window_mark_plugin._head, &config->processor_plugins); + mk_list_add(&single_mark_plugin._head, &config->processor_plugins); + mk_list_add(&window_drop_plugin._head, &config->processor_plugins); + + return config; +} + +static void window_config_destroy(struct flb_config *config) +{ + mk_list_del(&window_mark_plugin._head); + mk_list_del(&single_mark_plugin._head); + mk_list_del(&window_drop_plugin._head); + flb_config_exit(config); +} + +static struct flb_processor *window_processor_create(struct flb_config *config, + struct flb_input_instance *owner, + char *first, char *second) +{ + struct flb_processor *proc; + + proc = flb_processor_create(config, "window_test", NULL, 0); + if (proc == NULL) { + return NULL; + } + proc->data = owner; + proc->source_plugin_type = FLB_PLUGIN_INPUT; + + if (flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, first) == NULL || + (second && flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, second) == NULL) || + flb_processor_init(proc) != 0) { + flb_processor_destroy(proc); + return NULL; + } + + return proc; +} + +/* checks order, the processor mark and that every group stayed whole */ +static void check_window_output(char *buf, size_t size) +{ + int ret; + int type; + int seq = 0; + int groups = 0; + int in_group = FLB_FALSE; + int group_records = 0; + char name[8]; + msgpack_object *value; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + + ret = flb_log_event_decoder_init(&decoder, buf, size); + TEST_ASSERT(ret == FLB_EVENT_DECODER_SUCCESS); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + + if (type == FLB_LOG_EVENT_GROUP_START) { + TEST_CHECK(in_group == FLB_FALSE); + snprintf(name, sizeof(name), "g%d", groups); + value = &event.metadata->via.map.ptr[0].val; + TEST_CHECK(event.metadata->via.map.size == 1); + TEST_CHECK(value->type == MSGPACK_OBJECT_STR && + value->via.str.size == strlen(name) && + memcmp(value->via.str.ptr, name, strlen(name)) == 0); + in_group = FLB_TRUE; + group_records = 0; + continue; + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + TEST_CHECK(in_group == FLB_TRUE); + TEST_CHECK(group_records == WINDOW_GROUP_RECORDS); + in_group = FLB_FALSE; + groups++; + continue; + } + + if (in_group) { + group_records++; + } + TEST_CHECK(event.body->type == MSGPACK_OBJECT_MAP && + event.body->via.map.size == 3); + TEST_CHECK(event.body->via.map.ptr[0].val.via.i64 == seq); + TEST_CHECK(event.body->via.map.ptr[2].val.via.i64 == 1); + seq++; + } + + TEST_CHECK(seq == WINDOW_TOTAL_RECORDS); + TEST_CHECK(groups == WINDOW_GROUPS); + TEST_CHECK(in_group == FLB_FALSE); + flb_log_event_decoder_destroy(&decoder); +} + +static void processor_log_windows_match_single_chunk(void) +{ + int ret; + int records; + int single_calls; + char *input; + size_t input_size; + void *windowed; + size_t windowed_size; + void *single; + size_t single_size; + double items_in; + double items_out; + double invocations; + char stage_label[32]; + struct flb_config *config; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_input_instance owner; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + TEST_ASSERT(create_window_records(&input, &input_size) == 0); + TEST_CHECK(input_size > 4 * 65536); + + /* the single chunk reference */ + proc = window_processor_create(config, &owner, "test_single_mark", NULL); + TEST_ASSERT(proc != NULL); + window_calls = 0; + window_max_records = 0; + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &single, &single_size); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls == 1); + TEST_CHECK(window_max_records == WINDOW_TOTAL_RECORDS); + single_calls = window_calls; + flb_processor_destroy(proc); + + /* the windowed run must produce the same bytes */ + proc = window_processor_create(config, &owner, "test_window_mark", NULL); + TEST_ASSERT(proc != NULL); + window_calls = 0; + window_max_records = 0; + records = -1; + ret = flb_processor_run_with_count(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &windowed, &windowed_size, + -1, &records); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls > single_calls); + /* windows hold at most the record limit plus one whole group */ + TEST_CHECK(window_max_records <= 1024 + WINDOW_GROUP_RECORDS); + TEST_CHECK(records == WINDOW_TOTAL_RECORDS); + TEST_CHECK(windowed != NULL && windowed != input); + TEST_CHECK(windowed_size == single_size); + TEST_CHECK(windowed_size == single_size && + memcmp(windowed, single, single_size) == 0); + check_window_output(windowed, windowed_size); + + /* one invocation per buffer, with logical record counts */ + pu = mk_list_entry(proc->logs.next, struct flb_processor_unit, _head); + snprintf(stage_label, sizeof(stage_label), "%zu", pu->stage); + TEST_CHECK(get_counter_value_5(proc->cmt_invocations, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", + &invocations) == 0); + TEST_CHECK(get_counter_value_5(proc->cmt_items_in, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", + &items_in) == 0); + TEST_CHECK(get_counter_value_5(proc->cmt_items_out, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", + &items_out) == 0); + TEST_CHECK(invocations == 1.0); + TEST_CHECK(items_in == WINDOW_TOTAL_RECORDS); + TEST_CHECK(items_out == WINDOW_TOTAL_RECORDS); + flb_processor_destroy(proc); + + flb_free(windowed); + flb_free(single); + flb_free(input); + cmt_destroy(owner.cmt); + window_config_destroy(config); +} + +static void processor_log_windows_drop_all(void) +{ + int ret; + int records; + char *input; + size_t input_size; + void *output; + size_t output_size; + double items_drop; + char stage_label[32]; + struct flb_config *config; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_input_instance owner; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + TEST_ASSERT(create_window_records(&input, &input_size) == 0); + + proc = window_processor_create(config, &owner, "test_window_drop", NULL); + TEST_ASSERT(proc != NULL); + window_calls = 0; + records = -1; + output = input; + output_size = input_size; + ret = flb_processor_run_with_count(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &output, &output_size, + -1, &records); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls > 1); + TEST_CHECK(output == NULL); + TEST_CHECK(output_size == 0); + TEST_CHECK(records == 0); + + pu = mk_list_entry(proc->logs.next, struct flb_processor_unit, _head); + snprintf(stage_label, sizeof(stage_label), "%zu", pu->stage); + TEST_CHECK(get_counter_value_5(proc->cmt_items_drop, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", + &items_drop) == 0); + TEST_CHECK(items_drop == WINDOW_TOTAL_RECORDS); + + flb_processor_destroy(proc); + flb_free(input); + cmt_destroy(owner.cmt); + window_config_destroy(config); +} + +static void processor_log_windows_need_every_unit(void) +{ + int i; + int ret; + char *input; + size_t input_size; + void *output; + size_t output_size; + struct flb_config *config; + struct flb_processor *proc; + struct flb_input_instance owner; + struct flb_log_event_encoder *encoder; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + TEST_ASSERT(create_window_records(&input, &input_size) == 0); + + /* a unit without the capability keeps the whole run on one chunk */ + proc = window_processor_create(config, &owner, "test_window_mark", + "test_single_mark"); + TEST_ASSERT(proc != NULL); + window_calls = 0; + window_max_records = 0; + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &output, &output_size); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls == 2); + TEST_CHECK(window_max_records == WINDOW_TOTAL_RECORDS); + flb_free(output); + flb_processor_destroy(proc); + + flb_free(input); + + /* a buffer within one window keeps the single chunk path */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (i = 0; i < 300; i++) { + TEST_ASSERT(append_window_record(encoder, i) == 0); + } + proc = window_processor_create(config, &owner, "test_window_mark", NULL); + TEST_ASSERT(proc != NULL); + window_calls = 0; + window_max_records = 0; + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, + &output, &output_size); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls == 1); + TEST_CHECK(window_max_records == 300); + flb_free(output); + flb_processor_destroy(proc); + flb_log_event_encoder_destroy(encoder); + + cmt_destroy(owner.cmt); + window_config_destroy(config); +} + TEST_LIST = { {"processor_metrics_failure_clears_output", processor_metrics_failure_clears_output}, { "processor_private_inputs_use_main_loop", processor_private_inputs_use_main_loop }, { "processor", processor }, { "processor_grouped_filter_counters", processor_grouped_filter_counters }, { "processor_metrics_counters", processor_metrics_counters }, + { "processor_log_windows_match_single_chunk", processor_log_windows_match_single_chunk }, + { "processor_log_windows_drop_all", processor_log_windows_drop_all }, + { "processor_log_windows_need_every_unit", processor_log_windows_need_every_unit }, { 0 } }; From 8e9e4512a43e9ac38c4f964f9741481e759a37bf Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 20:36:21 -0600 Subject: [PATCH 26/61] tests: integration: cover windowed output processing Signed-off-by: Eduardo Silva --- .../processor_fts/tests/test_fts_pipeline.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py b/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py index 95afa656c69..a598ef69acc 100644 --- a/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py @@ -298,3 +298,74 @@ def test_pooled_batch_commit_rollback_and_retained_values(tmp_path, placement, r assert ("cannot construct policy output" in diagnostics) is rollback finally: service.stop() + + +@pytest.mark.parametrize("chain", ["direct", "native"]) +def test_output_windows_keep_otlp_resources_whole(tmp_path, chain): + """Large output chunks are processed in windows without splitting resources.""" + processors = [{"name": "fts", "code": '''function process(log) + log.body:set("seen", true) + log.resource.attributes:set("windowed", "yes") + end'''}] + if chain == "native": + processors.append({"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"}) + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + output = {"name": "stdout", "match": "*", "format": "otlp_json", + "processors": {"logs": processors}} + config = tmp_path / "windows.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 1, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + resources = 4 + per_resource = 1500 + service = FluentBitTestService(str(config)) + try: + service.start() + payload = {"resourceLogs": [{ + "resource": {"attributes": [{"key": "res.id", "value": {"stringValue": f"r{res}"}}]}, + "scopeLogs": [{"scope": {"name": "windows"}, "logRecords": [ + {"timeUnixNano": str(1700000000000000000 + index), + "body": {"kvlistValue": {"values": [ + {"key": "index", "value": {"intValue": str(index)}}, + {"key": "text", "value": {"stringValue": "w" * 48}}]}}} + for index in range(per_resource)]}]} for res in range(resources)]} + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/logs", + json=payload, timeout=30) + response.raise_for_status() + + def collect(): + seen = {} + batches = [] + for line in Path(service.flb.log_file).read_text().splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + batch = json.loads(line) + batches.append(batch) + for resource in batch["resourceLogs"]: + res_id = attributes(resource["resource"]["attributes"])["res.id"] + for scope in resource["scopeLogs"]: + seen.setdefault(res_id, []).extend(scope["logRecords"]) + total = sum(len(logs) for logs in seen.values()) + return (seen, batches) if total >= resources * per_resource else None + + seen, batches = service.wait_for_condition(collect, timeout=60) + assert sorted(seen) == [f"r{res}" for res in range(resources)] + for logs in seen.values(): + bodies = [scalar(log["body"]) for log in logs] + assert [body["index"] for body in bodies] == list(range(per_resource)) + assert all(body["seen"] is True and body["text"] == "w" * 48 for body in bodies) + assert all(body.get("following") == ("native" if chain == "native" else None) + for body in bodies) + for batch in batches: + ids = [attributes(resource["resource"]["attributes"])["res.id"] + for resource in batch["resourceLogs"]] + # a resource split across windows would be emitted more than once + assert len(ids) == len(set(ids)) + for resource in batch["resourceLogs"]: + assert attributes(resource["resource"]["attributes"])["windowed"] == "yes" + diagnostics = Path(service.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + assert "invalid log window" not in diagnostics + finally: + service.stop() From 4350195cce592ca21b31b3231696c71de0655bed Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:03:42 -0600 Subject: [PATCH 27/61] mp: drop unused log scanner modes flb_mp_scan_log_records() only counts records now: its typed validation and plain output have no remaining users. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_mp.h | 7 ++++--- src/flb_mp.c | 30 +++++++++--------------------- 2 files changed, 13 insertions(+), 24 deletions(-) diff --git a/include/fluent-bit/flb_mp.h b/include/fluent-bit/flb_mp.h index d778f47cc01..39fd21de487 100644 --- a/include/fluent-bit/flb_mp.h +++ b/include/fluent-bit/flb_mp.h @@ -29,10 +29,11 @@ int flb_mp_count(const void *data, size_t bytes); /* - * Allocation-free in non-tracking builds; rejects unsupported/deep shapes. - * 'typed' requires string map keys and CFL-compatible value types/depth. + * Count log records without building msgpack objects; allocation-free in + * non-tracking builds. Rejects unsupported or deep shapes, so callers can fall + * back to the event decoder. */ -int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *records, int *plain); +int flb_mp_scan_log_records(const void *data, size_t bytes, int *records); int flb_mp_count_log_records(const void *data, size_t bytes); int flb_mp_log_window_next(const void *data, size_t bytes, size_t offset, size_t limit_bytes, int limit_records, diff --git a/src/flb_mp.c b/src/flb_mp.c index 53231c60fd0..d9d0677d7c9 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -52,7 +52,7 @@ int flb_mp_count(const void *data, size_t bytes) * tree or zone is constructed. Unusual or deeper inputs are rejected so the * counting API can fall back to the event decoder. */ -static int scan_log_value(mpack_reader_t *reader, mpack_tag_t tag, int depth, int typed) +static int scan_log_value(mpack_reader_t *reader, mpack_tag_t tag, int depth) { int ret; uint32_t index; @@ -67,7 +67,7 @@ static int scan_log_value(mpack_reader_t *reader, mpack_tag_t tag, int depth, in case mpack_type_array: count = mpack_tag_array_count(&tag); for (index = 0; index < count; index++) { - ret = scan_log_value(reader, mpack_read_tag(reader), depth + 1, typed); + ret = scan_log_value(reader, mpack_read_tag(reader), depth + 1); if (ret != 0) { return -1; } @@ -78,17 +78,11 @@ static int scan_log_value(mpack_reader_t *reader, mpack_tag_t tag, int depth, in count = mpack_tag_map_count(&tag); for (index = 0; index < count; index++) { key = mpack_read_tag(reader); - - /* typed mode requires string keys */ - if (typed && mpack_tag_type(&key) != mpack_type_str) { - return -1; - } - - if (scan_log_value(reader, key, depth + 1, 0) != 0) { + if (scan_log_value(reader, key, depth + 1) != 0) { return -1; } - ret = scan_log_value(reader, mpack_read_tag(reader), depth + 1, typed); + ret = scan_log_value(reader, mpack_read_tag(reader), depth + 1); if (ret != 0) { return -1; } @@ -104,9 +98,6 @@ static int scan_log_value(mpack_reader_t *reader, mpack_tag_t tag, int depth, in mpack_done_bin(reader); break; case mpack_type_ext: - if (typed) { - return -1; - } mpack_skip_bytes(reader, mpack_tag_ext_length(&tag)); mpack_done_ext(reader); break; @@ -121,7 +112,7 @@ static int scan_log_value(mpack_reader_t *reader, mpack_tag_t tag, int depth, in return 0; } -int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *records, int *plain) +int flb_mp_scan_log_records(const void *data, size_t bytes, int *records) { mpack_reader_t reader; mpack_tag_t tag; @@ -130,7 +121,6 @@ int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *reco char extension[8]; int header; int count = 0; - int ungrouped = FLB_TRUE; int ret = -1; if (!data && bytes) { @@ -184,7 +174,7 @@ int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *reco if (header) { tag = mpack_read_tag(&reader); if (mpack_tag_type(&tag) != mpack_type_map || - scan_log_value(&reader, tag, 0, typed) != 0) { + scan_log_value(&reader, tag, 0) != 0) { goto done; } mpack_done_array(&reader); @@ -193,7 +183,7 @@ int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *reco /* body */ tag = mpack_read_tag(&reader); if (mpack_tag_type(&tag) != mpack_type_map || - scan_log_value(&reader, tag, 0, typed) != 0) { + scan_log_value(&reader, tag, 0) != 0) { goto done; } mpack_done_array(&reader); @@ -205,7 +195,7 @@ int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *reco /* group markers are not counted as records */ if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_START || time.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { - ungrouped = FLB_FALSE; + continue; } else if (flb_time_is_valid_eventtime(&time) == FLB_TRUE && count < INT_MAX) { count++; @@ -216,7 +206,6 @@ int flb_mp_scan_log_records(const void *data, size_t bytes, int typed, int *reco } *records = count; - *plain = ungrouped; ret = 0; done: @@ -237,11 +226,10 @@ int flb_mp_count_log_records(const void *data, size_t bytes) { int count; int ret; - int plain; struct flb_log_event log_event; struct flb_log_event_decoder decoder; - if (flb_mp_scan_log_records(data, bytes, FLB_FALSE, &count, &plain) == 0) { + if (flb_mp_scan_log_records(data, bytes, &count) == 0) { return count; } From 90b63a9c57ec0a980c35a0fc2c23010aee6728c3 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:35:30 -0600 Subject: [PATCH 28/61] tests: internal: mp: use the record-only log scanner Signed-off-by: Eduardo Silva --- tests/internal/mp.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/internal/mp.c b/tests/internal/mp.c index 4aa4461e320..89938634555 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -770,7 +770,6 @@ static void test_log_scan_matches_decoder(void) size_t index; int ret; int count; - int plain; int record; char saved; @@ -793,8 +792,8 @@ static void test_log_scan_matches_decoder(void) } } TEST_CHECK(flb_mp_scan_log_records(encoder->output_buffer, encoder->output_length, - FLB_TRUE, &count, &plain) == 0); - TEST_CHECK(count == 4 && plain == FLB_FALSE); + &count) == 0); + TEST_CHECK(count == 4); for (length = 0; length <= encoder->output_length; length++) { TEST_CHECK(flb_mp_count_log_records(encoder->output_buffer, length) == reference_log_count(encoder->output_buffer, length)); From 692c9ad9fcf80e554b6904b62e8948f5adc23c5a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:03:42 -0600 Subject: [PATCH 29/61] mp: keep end markers in the previous log window A window could close right before a group end marker, so the next window started with an end that looked like the end of a group opened by a previous append. The scanner now extends the window over it. Early exits also flag the reader error, so unfinished compound types are not reported by tracking builds. Signed-off-by: Eduardo Silva --- src/flb_mp.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/src/flb_mp.c b/src/flb_mp.c index d9d0677d7c9..1d137d2c847 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -250,6 +250,47 @@ int flb_mp_count_log_records(const void *data, size_t bytes) return count; } +/* TRUE when the event starting at 'data' is a group end marker */ +static int log_event_is_group_end(const char *data, size_t bytes) +{ + mpack_reader_t reader; + mpack_tag_t tag; + msgpack_object timestamp = {0}; + struct flb_time time; + char extension[8]; + int result = FLB_FALSE; + + mpack_reader_init_data(&reader, data, bytes); + + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) == mpack_type_array && mpack_tag_array_count(&tag) == 2) { + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) == mpack_type_array && mpack_tag_array_count(&tag) == 2) { + tag = mpack_read_tag(&reader); + } + + /* group markers always use the EventTime extension */ + if (mpack_tag_type(&tag) == mpack_type_ext && + mpack_tag_ext_length(&tag) == 8 && mpack_tag_ext_exttype(&tag) == 0) { + mpack_read_bytes(&reader, extension, sizeof(extension)); + if (mpack_reader_error(&reader) == mpack_ok) { + timestamp.type = MSGPACK_OBJECT_EXT; + timestamp.via.ext.ptr = extension; + timestamp.via.ext.size = sizeof(extension); + timestamp.via.ext.type = 0; + result = flb_log_event_decoder_decode_timestamp(×tamp, &time) == 0 && + time.tm.tv_sec == FLB_LOG_EVENT_GROUP_END; + } + } + } + + /* only the event header was read */ + mpack_reader_flag_error(&reader, mpack_error_invalid); + mpack_reader_destroy(&reader); + + return result; +} + /* * Find the end of the log window starting at 'offset'. A window closes after * 'limit_bytes' of serialized events or 'limit_records' log records, but only @@ -345,7 +386,15 @@ int flb_mp_log_window_next(const void *data, size_t bytes, size_t offset, } position = (bytes - offset) - mpack_reader_remaining(&reader, NULL); - if (!in_group && (position >= limit_bytes || records >= limit_records)) { + + /* + * Never start the next window with an end marker: per window it would + * look like the end of a group opened by a previous append. + */ + if (!in_group && (position >= limit_bytes || records >= limit_records) && + (offset + position >= bytes || + !log_event_is_group_end((const char *) data + offset + position, + bytes - offset - position))) { break; } } @@ -357,6 +406,10 @@ int flb_mp_log_window_next(const void *data, size_t bytes, size_t offset, } done: + /* flag early exits so unfinished compound types are not reported */ + if (ret != 0) { + mpack_reader_flag_error(&reader, mpack_error_invalid); + } if (mpack_reader_destroy(&reader) != mpack_ok) { ret = -1; } From 9d7fad03ccf26493473a320cc4c0d23d6720a6f0 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:03:42 -0600 Subject: [PATCH 30/61] mp: keep group markers that span appends on request Log group normalization removes every unterminated group start and every end marker without a start, so a group whose start and end arrive in separate appends loses its markers whenever a native processor runs. Add flags to keep the first end marker before any start and a single group still open at the end, and flb_mp_log_group_boundaries() to report which of those markers an input buffer has. The existing normalization functions keep their strict behaviour. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_mp.h | 9 ++ include/fluent-bit/flb_mp_chunk.h | 1 + src/flb_mp.c | 178 +++++++++++++++++++++++++++++- 3 files changed, 186 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/flb_mp.h b/include/fluent-bit/flb_mp.h index 39fd21de487..d979362f736 100644 --- a/include/fluent-bit/flb_mp.h +++ b/include/fluent-bit/flb_mp.h @@ -38,8 +38,17 @@ int flb_mp_count_log_records(const void *data, size_t bytes); int flb_mp_log_window_next(const void *data, size_t bytes, size_t offset, size_t limit_bytes, int limit_records, size_t *window_end, int *window_records); +/* group markers that belong to neighbouring appends, see flb_mp_log_group_boundaries() */ +#define FLB_MP_GROUPS_KEEP_LEADING_END 1 +#define FLB_MP_GROUPS_KEEP_TRAILING_OPEN 2 + +int flb_mp_log_group_boundaries(const void *data, size_t bytes); +int flb_mp_log_group_starts(const void *data, size_t bytes); int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_size, char **out_buf, size_t *out_size); +int flb_mp_normalize_log_buffer_groups_msgpack_ex(const void *in_buf, size_t in_size, + char **out_buf, size_t *out_size, + int flags); int flb_mp_normalize_log_buffer_groups(const void *in_buf, size_t in_size, char **out_buf, size_t *out_size); int flb_mp_count_remaining(const void *data, size_t bytes, size_t *remaining_bytes); diff --git a/include/fluent-bit/flb_mp_chunk.h b/include/fluent-bit/flb_mp_chunk.h index 58d8fc0233d..0ab5680e67b 100644 --- a/include/fluent-bit/flb_mp_chunk.h +++ b/include/fluent-bit/flb_mp_chunk.h @@ -93,6 +93,7 @@ int flb_mp_chunk_cobj_encode_append(struct flb_mp_chunk_cobj *chunk_cobj); int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size); int flb_mp_chunk_cobj_count_log_records(struct flb_mp_chunk_cobj *chunk_cobj); int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj); +int flb_mp_chunk_cobj_normalize_groups_ex(struct flb_mp_chunk_cobj *chunk_cobj, int flags); diff --git a/src/flb_mp.c b/src/flb_mp.c index 1d137d2c847..6570ab8ead2 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -250,6 +250,131 @@ int flb_mp_count_log_records(const void *data, size_t bytes) return count; } +/* + * Report the group markers of a log buffer that belong to neighbouring + * appends: an end marker before any start (FLB_MP_GROUPS_KEEP_LEADING_END) and + * a single group still open at the end (FLB_MP_GROUPS_KEEP_TRAILING_OPEN). + * Returns the flags, or 0 when the buffer cannot be scanned. + */ +static int log_group_scan(const void *data, size_t bytes, int *starts); + +int flb_mp_log_group_boundaries(const void *data, size_t bytes) +{ + return log_group_scan(data, bytes, NULL); +} + +/* number of group start markers in a log buffer, 0 when it cannot be scanned */ +int flb_mp_log_group_starts(const void *data, size_t bytes) +{ + int starts = 0; + + log_group_scan(data, bytes, &starts); + return starts; +} + +static int log_group_scan(const void *data, size_t bytes, int *starts) +{ + mpack_reader_t reader; + mpack_tag_t tag; + msgpack_object timestamp = {0}; + struct flb_time time; + char extension[8]; + int header; + int flags = 0; + int depth = 0; + int seen_start = FLB_FALSE; + int failed = FLB_TRUE; + + if (!data || bytes == 0) { + return 0; + } + + mpack_reader_init_data(&reader, data, bytes); + + while (mpack_reader_remaining(&reader, NULL) > 0) { + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_array || mpack_tag_array_count(&tag) != 2) { + goto done; + } + + tag = mpack_read_tag(&reader); + header = mpack_tag_type(&tag) == mpack_type_array; + if (header) { + if (mpack_tag_array_count(&tag) != 2) { + goto done; + } + tag = mpack_read_tag(&reader); + } + + /* only group markers matter, and they use the EventTime extension */ + timestamp.type = MSGPACK_OBJECT_POSITIVE_INTEGER; + if (mpack_tag_type(&tag) == mpack_type_ext && + mpack_tag_ext_length(&tag) == 8 && mpack_tag_ext_exttype(&tag) == 0) { + mpack_read_bytes(&reader, extension, sizeof(extension)); + mpack_done_ext(&reader); + timestamp.type = MSGPACK_OBJECT_EXT; + timestamp.via.ext.ptr = extension; + timestamp.via.ext.size = sizeof(extension); + timestamp.via.ext.type = 0; + } + else if (mpack_tag_type(&tag) == mpack_type_ext) { + mpack_skip_bytes(&reader, mpack_tag_ext_length(&tag)); + mpack_done_ext(&reader); + } + else if (mpack_tag_type(&tag) == mpack_type_array || + mpack_tag_type(&tag) == mpack_type_map || + mpack_tag_type(&tag) == mpack_type_str || + mpack_tag_type(&tag) == mpack_type_bin) { + goto done; + } + + if (header) { + mpack_discard(&reader); + mpack_done_array(&reader); + } + mpack_discard(&reader); + mpack_done_array(&reader); + + if (mpack_reader_error(&reader) != mpack_ok) { + goto done; + } + + if (timestamp.type != MSGPACK_OBJECT_EXT || + flb_log_event_decoder_decode_timestamp(×tamp, &time) != 0) { + continue; + } + + if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_START) { + seen_start = FLB_TRUE; + depth++; + if (starts) { + (*starts)++; + } + } + else if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { + if (depth > 0) { + depth--; + } + else if (!seen_start) { + flags |= FLB_MP_GROUPS_KEEP_LEADING_END; + } + } + } + + if (depth == 1) { + flags |= FLB_MP_GROUPS_KEEP_TRAILING_OPEN; + } + failed = FLB_FALSE; + +done: + if (failed) { + mpack_reader_flag_error(&reader, mpack_error_invalid); + } + mpack_reader_destroy(&reader); + + return failed ? 0 : flags; +} + /* TRUE when the event starting at 'data' is a group end marker */ static int log_event_is_group_end(const char *data, size_t bytes) { @@ -426,11 +551,28 @@ struct flb_mp_record_span { int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_size, char **out_buf, size_t *out_size) +{ + return flb_mp_normalize_log_buffer_groups_msgpack_ex(in_buf, in_size, out_buf, out_size, 0); +} + +/* + * Group normalization removes empty groups, end markers without a start and + * groups left open at the end of the buffer. 'flags' keeps the markers at the + * append boundaries when the caller knows the input already had them: a group + * split across appends keeps its first end marker before any start + * (FLB_MP_GROUPS_KEEP_LEADING_END) and its single open trailing group + * (FLB_MP_GROUPS_KEEP_TRAILING_OPEN). + */ +int flb_mp_normalize_log_buffer_groups_msgpack_ex(const void *in_buf, size_t in_size, + char **out_buf, size_t *out_size, + int flags) { int i; int ret; int entry_count; int stack_count; + int seen_start = FLB_FALSE; + int leading_end = FLB_FALSE; int *group_stack; int *group_has_content; struct flb_log_event log_event; @@ -518,12 +660,19 @@ int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_siz group_stack[stack_count] = i; group_has_content[stack_count] = FLB_FALSE; stack_count++; + seen_start = FLB_TRUE; continue; } if (entries[i].type == FLB_LOG_EVENT_GROUP_END) { if (stack_count == 0) { - entries[i].keep = FLB_FALSE; + /* the first end before any start closes a previous append's group */ + if ((flags & FLB_MP_GROUPS_KEEP_LEADING_END) && !seen_start && !leading_end) { + leading_end = FLB_TRUE; + } + else { + entries[i].keep = FLB_FALSE; + } continue; } @@ -545,6 +694,11 @@ int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_siz } } + /* a single open group continues in the next append, nested ones are dropped */ + if ((flags & FLB_MP_GROUPS_KEEP_TRAILING_OPEN) && stack_count == 1) { + stack_count = 0; + } + while (stack_count > 0) { stack_count--; entries[group_stack[stack_count]].keep = FLB_FALSE; @@ -2041,9 +2195,17 @@ static int cobj_remove_list_add(struct flb_mp_chunk_record **remove_list, } int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj) +{ + return flb_mp_chunk_cobj_normalize_groups_ex(chunk_cobj, 0); +} + +/* see flb_mp_normalize_log_buffer_groups_msgpack_ex() for 'flags' */ +int flb_mp_chunk_cobj_normalize_groups_ex(struct flb_mp_chunk_cobj *chunk_cobj, int flags) { int ret; int record_type; + int seen_start = FLB_FALSE; + int leading_end = FLB_FALSE; size_t index; size_t stack_size; size_t stack_count; @@ -2101,12 +2263,19 @@ int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj) group_has_content[stack_count] = FLB_FALSE; stack_count++; } + seen_start = FLB_TRUE; continue; } if (record_type == FLB_LOG_EVENT_GROUP_END) { if (stack_count == 0) { - cobj_remove_list_add(remove_list, stack_size, &remove_count, record); + /* the first end before any start closes a previous append's group */ + if ((flags & FLB_MP_GROUPS_KEEP_LEADING_END) && !seen_start && !leading_end) { + leading_end = FLB_TRUE; + } + else { + cobj_remove_list_add(remove_list, stack_size, &remove_count, record); + } continue; } @@ -2129,6 +2298,11 @@ int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj) } } + /* a single open group continues in the next append, nested ones are dropped */ + if ((flags & FLB_MP_GROUPS_KEEP_TRAILING_OPEN) && stack_count == 1) { + stack_count = 0; + } + while (stack_count > 0) { stack_count--; cobj_remove_list_add(remove_list, stack_size, &remove_count, From 54bc2af353735621891e081a3934f10699613834 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:04:05 -0600 Subject: [PATCH 31/61] tests: internal: mp: cover group normalization and window boundaries Signed-off-by: Eduardo Silva --- tests/internal/mp.c | 209 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) diff --git a/tests/internal/mp.c b/tests/internal/mp.c index 89938634555..bc43203944b 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -885,6 +886,8 @@ static void test_log_window_next(void) TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, group_start, 1, INT_MAX, &end, &records) == 0); TEST_CHECK(end == group_end && records == 6); + TEST_CHECK(flb_mp_log_group_starts(encoder->output_buffer + group_start, + group_end - group_start) == 1); /* a single window for the whole buffer */ TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, @@ -900,11 +903,217 @@ static void test_log_window_next(void) TEST_CHECK(flb_mp_log_window_next("\x92\x01", 2, 0, 1, 1, &end, &records) == -1); flb_log_event_encoder_destroy(encoder); + + /* a full window keeps a following end marker instead of starting with it */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (seq = 0; seq < 4; seq++) { + append_window_test_record(encoder, seq); + } + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + group_end = encoder->output_length; + TEST_ASSERT(flb_log_event_encoder_group_init(encoder) == 0); + TEST_ASSERT(flb_log_event_encoder_group_header_end(encoder) == 0); + append_window_test_record(encoder, seq); + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + 0, SIZE_MAX, 4, &end, &records) == 0); + TEST_CHECK(end == group_end && records == 4); + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + end, SIZE_MAX, 4, &end, &records) == 0); + TEST_CHECK(end == encoder->output_length && records == 1); + + flb_log_event_encoder_destroy(encoder); +} + +/* layout tokens: r record, S group start, E group end */ +static void encode_group_layout(struct flb_log_event_encoder *encoder, const char *layout) +{ + int length; + char token[16]; + const char *cursor; + + for (cursor = layout; sscanf(cursor, "%15s%n", token, &length) == 1; cursor += length) { + if (token[0] == 'r') { + append_window_test_record(encoder, atoi(token + 1)); + } + else if (token[0] == 'S') { + TEST_CHECK(flb_log_event_encoder_group_init(encoder) == 0); + TEST_CHECK(flb_log_event_encoder_group_header_end(encoder) == 0); + } + else { + TEST_CHECK(flb_log_event_encoder_group_end(encoder) == 0); + } + } +} + +static void describe_group_layout(char *buf, size_t size, char *out, size_t out_size) +{ + int type; + char token[16]; + msgpack_object *seq; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + + out[0] = '\0'; + if (buf == NULL || size == 0) { + return; + } + + TEST_ASSERT(flb_log_event_decoder_init(&decoder, buf, size) == 0); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type == FLB_LOG_EVENT_GROUP_START) { + snprintf(token, sizeof(token), "S "); + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + snprintf(token, sizeof(token), "E "); + } + else { + seq = &event.body->via.map.ptr[0].val; + snprintf(token, sizeof(token), "r%d ", (int) seq->via.i64); + } + strncat(out, token, out_size - strlen(out) - 1); + } + flb_log_event_decoder_destroy(&decoder); + if (strlen(out) > 0) { + out[strlen(out) - 1] = '\0'; + } +} + +static void check_normalized_layout(char *output, size_t output_size, + const char *layout, const char *expected, + const char *mode) +{ + char result[128]; + + describe_group_layout(output, output ? output_size : 0, result, sizeof(result)); + TEST_CHECK(strcmp(result, expected) == 0); + TEST_MSG("%s: '%s' produced '%s', expected '%s'", mode, layout, result, expected); +} + +/* + * 'strict' is the default normalization. 'kept' keeps the boundary markers + * the input already had, as processor chains do for groups split across + * appends. + */ +static void check_group_normalization(const char *layout, const char *strict, + const char *kept) +{ + struct flb_log_event_encoder *encoder; + struct flb_log_event_encoder *copy; + struct flb_mp_chunk_cobj *chunk; + struct flb_log_event_decoder *decoder; + struct flb_mp_chunk_record *record; + char *output; + size_t output_size; + int flags; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + encode_group_layout(encoder, layout); + flags = flb_mp_log_group_boundaries(encoder->output_buffer, encoder->output_length); + + output = NULL; + TEST_CHECK(flb_mp_normalize_log_buffer_groups_msgpack(encoder->output_buffer, + encoder->output_length, + &output, &output_size) == 0); + check_normalized_layout(output, output_size, layout, strict, "msgpack strict"); + flb_free(output); + + output = NULL; + TEST_CHECK(flb_mp_normalize_log_buffer_groups(encoder->output_buffer, + encoder->output_length, + &output, &output_size) == 0); + check_normalized_layout(output, output_size, layout, strict, "chunk strict"); + flb_free(output); + + output = NULL; + TEST_CHECK(flb_mp_normalize_log_buffer_groups_msgpack_ex(encoder->output_buffer, + encoder->output_length, + &output, &output_size, + flags) == 0); + check_normalized_layout(output, output_size, layout, kept, "msgpack kept"); + flb_free(output); + + /* the chunk normalizer with the same flags */ + decoder = flb_log_event_decoder_create(encoder->output_buffer, encoder->output_length); + TEST_ASSERT(decoder != NULL); + flb_log_event_decoder_read_groups(decoder, FLB_TRUE); + copy = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(copy != NULL); + chunk = flb_mp_chunk_cobj_create(copy, decoder); + TEST_ASSERT(chunk != NULL); + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + } + TEST_CHECK(flb_mp_chunk_cobj_normalize_groups_ex(chunk, flags) == 0); + output = NULL; + output_size = 0; + if (cfl_list_size(&chunk->records) > 0) { + TEST_CHECK(flb_mp_chunk_cobj_encode(chunk, &output, &output_size) == 0); + } + check_normalized_layout(output, output_size, layout, kept, "chunk kept"); + flb_free(output); + flb_mp_chunk_cobj_destroy(chunk); + flb_log_event_encoder_destroy(copy); + flb_log_event_decoder_destroy(decoder); + + flb_log_event_encoder_destroy(encoder); +} + +static void test_group_normalization_keeps_append_boundaries(void) +{ + /* a group continued in the next append keeps its start when kept */ + check_group_normalization("S r0 r1", "r0 r1", "S r0 r1"); + check_group_normalization("S", "", "S"); + /* records closing a group opened by a previous append keep their end */ + check_group_normalization("r0 r1 E", "r0 r1", "r0 r1 E"); + check_group_normalization("r0 E S r1 E S r2", "r0 S r1 E r2", "r0 E S r1 E S r2"); + /* other stray ends, empty groups and nested open groups are always removed */ + check_group_normalization("r0 E E S r1 E E r2", "r0 S r1 E r2", "r0 E S r1 E r2"); + check_group_normalization("S E r0", "r0", "r0"); + check_group_normalization("S S r0", "r0", "r0"); + check_group_normalization("S r0 E S E r1", "S r0 E r1", "S r0 E r1"); + check_group_normalization("S r0 E r1", "S r0 E r1", "S r0 E r1"); +} + +static void test_group_boundaries_follow_the_input(void) +{ + struct flb_log_event_encoder *encoder; + int flags; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + encode_group_layout(encoder, "r0 E S r1 E S r2"); + flags = flb_mp_log_group_boundaries(encoder->output_buffer, encoder->output_length); + TEST_CHECK(flags == (FLB_MP_GROUPS_KEEP_LEADING_END | FLB_MP_GROUPS_KEEP_TRAILING_OPEN)); + flb_log_event_encoder_destroy(encoder); + + /* a balanced input keeps nothing, whatever a processor removes later */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + encode_group_layout(encoder, "S r0 E r1"); + TEST_CHECK(flb_mp_log_group_boundaries(encoder->output_buffer, encoder->output_length) == 0); + flb_log_event_encoder_destroy(encoder); + + /* nested open groups are not a single continued group */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + encode_group_layout(encoder, "S S r0"); + TEST_CHECK(flb_mp_log_group_boundaries(encoder->output_buffer, encoder->output_length) == 0); + flb_log_event_encoder_destroy(encoder); + + TEST_CHECK(flb_mp_log_group_boundaries(NULL, 0) == 0); + TEST_CHECK(flb_mp_log_group_boundaries("\x92\x01", 2) == 0); } TEST_LIST = { {"log_scan_matches_decoder", test_log_scan_matches_decoder}, {"log_window_next", test_log_window_next}, + {"group_normalization_append_boundaries", test_group_normalization_keeps_append_boundaries}, + {"group_boundaries_follow_the_input", test_group_boundaries_follow_the_input}, {"preflight", test_preflight}, {"preflight_formats", test_preflight_formats}, {"validate_trace_chunk" , test_validate_trace_chunk}, From aa133607e15c12d0229ef91186a408ba962bdb85 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:03:51 -0600 Subject: [PATCH 32/61] processor: give each run its own chunk decoder and encoder The decoded chunk used the first native unit's decoder and encoder, but it lives on through later units while only their locks are held. Records are decoded lazily, so with output workers another thread could reset that decoder while a later unit was still reading from it, corrupting memory. Each processor run now owns its decoder and encoder, created on first use, for both single chunk and windowed processing. Signed-off-by: Eduardo Silva --- src/flb_processor.c | 133 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 109 insertions(+), 24 deletions(-) diff --git a/src/flb_processor.c b/src/flb_processor.c index 2937fd50507..3146b53ba47 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -1314,6 +1314,51 @@ int flb_processor_is_active(struct flb_processor *proc) * context for metrics, a 'CTraces' context for traces or a 'CProfiles' context for * profiles. */ +/* + * Decoder and encoder owned by one processor run. The decoded chunk lives + * across several units, so it must not borrow the first unit's decoder: its + * lock is released before the next units run and another worker thread + * could reset it while records are still being decoded. + */ +struct chunk_codec { + int ready; + struct flb_log_event_decoder decoder; + struct flb_log_event_encoder encoder; +}; + +static int chunk_codec_prepare(struct chunk_codec *codec) +{ + if (codec->ready) { + return 0; + } + + if (flb_log_event_decoder_init(&codec->decoder, NULL, 0) != FLB_EVENT_DECODER_SUCCESS) { + return -1; + } + /* chunks keep group markers as records, like processor instance decoders */ + flb_log_event_decoder_read_groups(&codec->decoder, FLB_TRUE); + + if (flb_log_event_encoder_init(&codec->encoder, + FLB_LOG_EVENT_FORMAT_DEFAULT) != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_decoder_destroy(&codec->decoder); + return -1; + } + + codec->ready = FLB_TRUE; + return 0; +} + +static void chunk_codec_destroy(struct chunk_codec *codec) +{ + if (!codec->ready) { + return; + } + + flb_log_event_decoder_destroy(&codec->decoder); + flb_log_event_encoder_destroy(&codec->encoder); + codec->ready = FLB_FALSE; +} + /* * Large log buffers, such as complete chunks reaching output processors, are * processed in windows when every native unit of the run is record local: @@ -1414,7 +1459,7 @@ static struct log_window *log_windows_scan(void *data, size_t data_size, int *co return windows; } -static int run_log_windows(struct flb_processor *proc, +static int run_log_windows(struct flb_processor *proc, struct chunk_codec *codec, struct mk_list *first, int unit_count, const char *tag, size_t tag_len, void *data, size_t data_size, @@ -1437,7 +1482,6 @@ static int run_log_windows(struct flb_processor *proc, struct log_window *windows; struct log_window_unit *units; struct flb_processor_instance *p_ins; - struct flb_processor_instance *owner; struct flb_mp_chunk_cobj *chunk_cobj = NULL; windows = log_windows_scan(data, data_size, &window_count); @@ -1452,20 +1496,23 @@ static int run_log_windows(struct flb_processor *proc, return LOG_WINDOWS_SKIP; } + if (chunk_codec_prepare(codec) != 0) { + flb_free(units); + flb_free(windows); + return LOG_WINDOWS_SKIP; + } + head = first; for (i = 0; i < unit_count; i++) { units[i].pu = mk_list_entry(head, struct flb_processor_unit, _head); head = head->next; } - /* the first unit owns the decoder and the encoder, as in a single chunk */ - owner = units[0].pu->ctx; - offset = 0; for (w = 0; w < window_count; w++) { - flb_log_event_decoder_reset(owner->log_decoder, (char *) data + offset, + flb_log_event_decoder_reset(&codec->decoder, (char *) data + offset, windows[w].end - offset); - chunk_cobj = flb_mp_chunk_cobj_create(owner->log_encoder, owner->log_decoder); + chunk_cobj = flb_mp_chunk_cobj_create(&codec->encoder, &codec->decoder); if (!chunk_cobj) { goto error; } @@ -1515,16 +1562,16 @@ static int run_log_windows(struct flb_processor *proc, offset = windows[w].end; } - if (total_records == 0 || owner->log_encoder->output_length == 0) { - flb_log_event_encoder_reset(owner->log_encoder); + if (total_records == 0 || codec->encoder.output_length == 0) { + flb_log_event_encoder_reset(&codec->encoder); *out_buf = NULL; *out_size = 0; total_records = 0; } else { - *out_buf = owner->log_encoder->output_buffer; - *out_size = owner->log_encoder->output_length; - flb_log_event_encoder_claim_internal_buffer_ownership(owner->log_encoder); + *out_buf = codec->encoder.output_buffer; + *out_size = codec->encoder.output_length; + flb_log_event_encoder_claim_internal_buffer_ownership(&codec->encoder); } *out_records = total_records; @@ -1536,10 +1583,10 @@ static int run_log_windows(struct flb_processor *proc, if (chunk_cobj) { flb_mp_chunk_cobj_destroy(chunk_cobj); } - flb_log_event_encoder_reset(owner->log_encoder); + flb_log_event_encoder_reset(&codec->encoder); exit: - flb_log_event_decoder_reset(owner->log_decoder, NULL, 0); + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); for (i = 0; i < unit_count; i++) { processor_metrics_update(proc, units[i].pu, @@ -1564,7 +1611,7 @@ int flb_processor_run(struct flb_processor *proc, data, data_size, out_buf, out_size, -1, NULL); } -int flb_processor_run_with_count(struct flb_processor *proc, +static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, size_t starting_stage, int type, const char *tag, size_t tag_len, @@ -1672,7 +1719,7 @@ int flb_processor_run_with_count(struct flb_processor *proc, cur_size > FLB_PROCESSOR_LOG_WINDOW_BYTES) { run_end = log_window_run_end(head, list, &run_units); if (run_end != NULL) { - ret = run_log_windows(proc, head, run_units, tag, tag_len, + ret = run_log_windows(proc, codec, head, run_units, tag, tag_len, cur_buf, cur_size, &tmp_buf, &tmp_size, &window_records, &window_normalized, metrics_scope, metrics_owner, metrics_signal); @@ -1861,12 +1908,30 @@ int flb_processor_run_with_count(struct flb_processor *proc, * until no more processors exists or the next one is a "filter" type processor. */ if (!chunk_cobj) { - flb_log_event_decoder_reset(p_ins->log_decoder, cur_buf, cur_size); + if (chunk_codec_prepare(codec) != 0) { + if (cur_buf != data) { + flb_free(cur_buf); + } + pu_error = FLB_TRUE; + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); + release_lock(&pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + return -1; + } + flb_log_event_decoder_reset(&codec->decoder, cur_buf, cur_size); /* create the context */ - chunk_cobj = flb_mp_chunk_cobj_create(p_ins->log_encoder, p_ins->log_decoder); + chunk_cobj = flb_mp_chunk_cobj_create(&codec->encoder, &codec->decoder); if (chunk_cobj == NULL) { - flb_log_event_decoder_reset(p_ins->log_decoder, NULL, 0); + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); if (cur_buf != data) { flb_free(cur_buf); } @@ -1938,7 +2003,7 @@ int flb_processor_run_with_count(struct flb_processor *proc, if (output_records) { *output_records = 0; } - flb_log_event_encoder_reset(p_ins->log_encoder); + flb_log_event_encoder_reset(&codec->encoder); flb_mp_chunk_cobj_destroy(chunk_cobj); *out_buf = NULL; @@ -1962,8 +2027,8 @@ int flb_processor_run_with_count(struct flb_processor *proc, /* encode chunk_cobj as msgpack */ ret = flb_mp_chunk_cobj_encode(chunk_cobj, (char **) &tmp_buf, &tmp_size); if (ret != 0) { - flb_log_event_decoder_reset(p_ins->log_decoder, NULL, 0); - flb_log_event_encoder_reset(p_ins->log_encoder); + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); + flb_log_event_encoder_reset(&codec->encoder); flb_mp_chunk_cobj_destroy(chunk_cobj); if (cur_buf != data) { @@ -1994,8 +2059,8 @@ int flb_processor_run_with_count(struct flb_processor *proc, cur_size = tmp_size; - flb_log_event_decoder_reset(p_ins->log_decoder, NULL, 0); - flb_log_event_encoder_claim_internal_buffer_ownership(p_ins->log_encoder); + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); + flb_log_event_encoder_claim_internal_buffer_ownership(&codec->encoder); flb_mp_chunk_cobj_destroy(chunk_cobj); chunk_cobj = NULL; } @@ -2205,6 +2270,26 @@ int flb_processor_run_with_count(struct flb_processor *proc, return 0; } +int flb_processor_run_with_count(struct flb_processor *proc, + size_t starting_stage, + int type, + const char *tag, size_t tag_len, + void *data, size_t data_size, + void **out_buf, size_t *out_size, + int input_records, int *output_records) +{ + int ret; + struct chunk_codec codec; + + codec.ready = FLB_FALSE; + ret = processor_run(proc, &codec, starting_stage, type, tag, tag_len, + data, data_size, out_buf, out_size, + input_records, output_records); + chunk_codec_destroy(&codec); + + return ret; +} + void flb_processor_destroy(struct flb_processor *proc) { struct mk_list *head; From 7c40058ad21a002aba4f8e34504b9acccb9a59db Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:00 -0600 Subject: [PATCH 33/61] processor: keep only the group boundaries the input already had Group normalization now keeps a leading end marker and a single open trailing group only on request. Processor runs ask for them only when their own input had them, so a group split across appends keeps its markers, while a processor that removes an end marker from a balanced input still loses the orphan start, as before. In windowed runs the leading end stays with the windows before the first group start and the open group with the last window. Signed-off-by: Eduardo Silva --- src/flb_processor.c | 72 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/src/flb_processor.c b/src/flb_processor.c index 3146b53ba47..836fe42f66e 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -1459,12 +1459,43 @@ static struct log_window *log_windows_scan(void *data, size_t data_size, int *co return windows; } +/* TRUE when a decoded chunk holds group markers */ +static int chunk_has_group_markers(struct flb_mp_chunk_cobj *chunk) +{ + int type; + struct cfl_list *head; + struct flb_mp_chunk_record *record; + + cfl_list_foreach(head, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + if (flb_log_event_decoder_get_record_type(&record->event, &type) == 0 && + (type == FLB_LOG_EVENT_GROUP_START || type == FLB_LOG_EVENT_GROUP_END)) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + +/* + * Group boundary flags of the processor input, scanned once and only when a + * chunk to normalize has group markers: plain logs never pay for the scan. + */ +static int input_group_flags(int *cache, void *data, size_t data_size) +{ + if (*cache < 0) { + *cache = flb_mp_log_group_boundaries(data, data_size); + } + return *cache; +} + static int run_log_windows(struct flb_processor *proc, struct chunk_codec *codec, struct mk_list *first, int unit_count, const char *tag, size_t tag_len, void *data, size_t data_size, void **out_buf, size_t *out_size, int *out_records, int *out_normalized, + void *input, size_t input_size, int *group_flags, const char *metrics_scope, const char *metrics_owner, const char *metrics_signal) @@ -1477,6 +1508,9 @@ static int run_log_windows(struct flb_processor *proc, struct chunk_codec *codec int total_records = 0; int normalized = FLB_TRUE; int window_count; + int window_flags; + int input_flags; + int seen_start = FLB_FALSE; size_t offset; struct mk_list *head; struct log_window *windows; @@ -1547,7 +1581,28 @@ static int run_log_windows(struct flb_processor *proc, struct chunk_codec *codec units[i].out_items += records; } - if (flb_mp_chunk_cobj_normalize_groups(chunk_cobj) != 0) { + /* + * Boundary markers of the input stay with the window holding them: a + * leading end until a group starts, the open group in the last window. + */ + window_flags = 0; + if (chunk_has_group_markers(chunk_cobj)) { + input_flags = input_group_flags(group_flags, input, input_size); + if (!seen_start) { + window_flags |= input_flags & FLB_MP_GROUPS_KEEP_LEADING_END; + } + if (w == window_count - 1) { + window_flags |= input_flags & FLB_MP_GROUPS_KEEP_TRAILING_OPEN; + } + } + /* only inputs opening with a continued group need the start count */ + if (*group_flags > 0 && (*group_flags & FLB_MP_GROUPS_KEEP_LEADING_END) && + !seen_start && + flb_mp_log_group_starts((char *) data + offset, windows[w].end - offset) > 0) { + seen_start = FLB_TRUE; + } + + if (flb_mp_chunk_cobj_normalize_groups_ex(chunk_cobj, window_flags) != 0) { normalized = FLB_FALSE; } total_records += flb_mp_chunk_cobj_count_log_records(chunk_cobj); @@ -1642,6 +1697,7 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, int run_units; int window_records; int window_normalized; + int group_flags = -1; const char *metrics_scope; const char *metrics_owner; const char *metrics_signal; @@ -1722,6 +1778,7 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, ret = run_log_windows(proc, codec, head, run_units, tag, tag_len, cur_buf, cur_size, &tmp_buf, &tmp_size, &window_records, &window_normalized, + data, data_size, &group_flags, metrics_scope, metrics_owner, metrics_signal); if (ret == LOG_WINDOWS_ERROR) { if (cur_buf != data) { @@ -1995,8 +2052,11 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, * count. Reuse them until a filter replaces this serialized * buffer. */ + /* keep only the boundary markers the input already had */ log_buffer_normalized = - flb_mp_chunk_cobj_normalize_groups(chunk_cobj) == 0; + flb_mp_chunk_cobj_normalize_groups_ex(chunk_cobj, + chunk_has_group_markers(chunk_cobj) ? + input_group_flags(&group_flags, data, data_size) : 0) == 0; log_buffer_records = flb_mp_chunk_cobj_count_log_records(chunk_cobj); if (cfl_list_size(&chunk_cobj->records) == 0) { @@ -2217,9 +2277,11 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, /* set output buffer */ if (type == FLB_PROCESSOR_LOGS && cur_buf != NULL && !log_buffer_normalized) { - ret = flb_mp_normalize_log_buffer_groups_msgpack(cur_buf, cur_size, - (char **) &tmp_buf, - &tmp_size); + ret = flb_mp_normalize_log_buffer_groups_msgpack_ex(cur_buf, cur_size, + (char **) &tmp_buf, &tmp_size, + input_group_flags(&group_flags, + data, + data_size)); if (ret == 0) { if (cur_buf != data) { flb_free(cur_buf); From b5848944d3a3e0ce1364eb402a169cce164ecfad Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:03:51 -0600 Subject: [PATCH 34/61] processor_fts: keep groups that span appends A group still open at the end of a batch was closed, and an end marker before any start was dropped, so records of a group split across appends moved out of it. Keep the open group open, reopening the source context when the last output group differs, and forward the leading end marker. Records before it keep no context and cannot set one. Signed-off-by: Eduardo Silva --- plugins/processor_fts/README.md | 6 ++ plugins/processor_fts/fts_logs.c | 141 ++++++++++++++++++++++++++++++- 2 files changed, 145 insertions(+), 2 deletions(-) diff --git a/plugins/processor_fts/README.md b/plugins/processor_fts/README.md index 0cbc3fc8aeb..7d548fb9a56 100644 --- a/plugins/processor_fts/README.md +++ b/plugins/processor_fts/README.md @@ -84,6 +84,12 @@ processors, are processed in batches of about 64 KiB or 1024 records, so decoded records never hold more than one batch in memory. Groups, such as one OpenTelemetry resource and scope, are never split across batches. +A group can also span appends, for example when its start and end markers +arrive in separate forward messages. A group still open at the end of a batch +stays open for the records that follow. Records that arrive before the end +marker of a group opened earlier are processed without a context, keep their +place in that group, and cannot set one. + For privacy policies such as redaction, keep the default `error_mode: drop`. With `keep`, an item whose policy fails is forwarded unmodified. diff --git a/plugins/processor_fts/fts_logs.c b/plugins/processor_fts/fts_logs.c index 93dc0716944..35e63223d95 100644 --- a/plugins/processor_fts/fts_logs.c +++ b/plugins/processor_fts/fts_logs.c @@ -1162,6 +1162,63 @@ static int serializes_after_callback(struct flb_processor_instance *ins) return unit->unit_type == FLB_PROCESSOR_UNIT_FILTER; } +/* + * Open a group with the source context of a group that continues in the next + * append, charging it to the output limit. OTLP transport IDs are fresh, so + * the group never merges with an earlier group of this chunk. + */ +static int reopen_source_group(struct fts_runtime *ctx, struct flb_mp_chunk_cobj *out, + struct cfl_object *source_attrs, + struct cfl_object *source_meta, + int resource_id, int scope_id, size_t *output_bytes) +{ + size_t cost; + struct cfl_object attrs = {0}; + struct cfl_object meta = {0}; + struct cfl_variant *id_value; + struct flb_mp_chunk_record *group = NULL; + + cost = 32; + if (*output_bytes > ctx->output_limit || ctx->output_limit - *output_bytes < cost) { + return -1; + } + *output_bytes += cost; + + attrs.type = CFL_OBJECT_KVLIST; + attrs.variant = source_attrs->variant; + meta.type = CFL_OBJECT_KVLIST; + meta.variant = source_meta ? source_meta->variant : NULL; + + cost = wire_value_size(attrs.variant, 0); + if (cost > ctx->output_limit || *output_bytes > ctx->output_limit - cost) { + return -1; + } + *output_bytes += cost; + + cost = wire_value_size(meta.variant, 0); + if (cost > ctx->output_limit || *output_bytes > ctx->output_limit - cost) { + return -1; + } + *output_bytes += cost; + + if (append_marker(out, FLB_LOG_EVENT_GROUP_START, &attrs, &meta, &group) != 0) { + return -1; + } + + if (has_otel_context(group->cobj_metadata)) { + id_value = cfl_variant_create_from_int64(resource_id); + if (set_transport_id(group->cobj_metadata->variant, "resource_id", id_value) != 0) { + return -1; + } + id_value = cfl_variant_create_from_int64(scope_id); + if (set_transport_id(group->cobj_metadata->variant, "scope_id", id_value) != 0) { + return -1; + } + } + + return 0; +} + int fts_process_logs(struct flb_processor_instance *ins, void *data, const char *tag, int tag_len) { @@ -1202,6 +1259,8 @@ int fts_process_logs(struct flb_processor_instance *ins, void *data, int base; int emitted_id = 0; int errors = 0; + int source_open = FLB_FALSE; + struct flb_mp_chunk_record *continuation_end = NULL; size_t undo_mark; struct cfl_variant *id_value; size_t output_bytes = 0; @@ -1235,6 +1294,25 @@ int fts_process_logs(struct flb_processor_instance *ins, void *data, cfl_list_init(&out->records); out->record_pool_enabled = FLB_TRUE; + /* + * Records before an end marker that precedes any start belong to a group + * opened by a previous append: they keep no context and the end marker is + * forwarded to close that group. + */ + cfl_list_foreach(head, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + if (flb_log_event_decoder_get_record_type(&record->event, &type) != 0) { + goto failure; + } + if (type == FLB_LOG_EVENT_GROUP_END) { + continuation_end = record; + break; + } + if (type == FLB_LOG_EVENT_GROUP_START) { + break; + } + } + cfl_list_foreach(head, &chunk->records) { record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); @@ -1247,14 +1325,28 @@ int fts_process_logs(struct flb_processor_instance *ins, void *data, /* A nested start establishes a new flat transport context. */ source_attrs = record->cobj_record; source_meta = record->cobj_metadata; + source_open = FLB_TRUE; group_number++; continue; } if (type == FLB_LOG_EVENT_GROUP_END) { + if (record == continuation_end) { + if (output_bytes > ctx->output_limit || ctx->output_limit - output_bytes < 32) { + goto failure; + } + output_bytes += 32; + if (append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + goto failure; + } + continuation_end = NULL; + continue; + } + /* Stray ends carry no record and do not invalidate healthy siblings. */ source_attrs = NULL; source_meta = NULL; + source_open = FLB_FALSE; continue; } @@ -1321,6 +1413,26 @@ int fts_process_logs(struct flb_processor_instance *ins, void *data, meta.variant = context_meta->owned; } } + + /* a group context would split the group continued from a previous append */ + if (continuation_end && attrs.variant) { + fts_undo_rollback(ctx, undo_mark); + errors++; + if (errors <= 5) { + flb_plg_error(ins, "log policy failed: records continuing a group " + "from a previous chunk cannot set a context; " + "edits rolled back"); + } + if (strcmp(ctx->error_mode, "drop") == 0) { + lua_settop(l, base); + continue; + } + ret = -1; + body = NULL; + metadata = NULL; + attrs.variant = NULL; + meta.variant = NULL; + } } /* Close the open group when the record's context no longer matches it */ @@ -1494,8 +1606,33 @@ int fts_process_logs(struct flb_processor_instance *ins, void *data, lua_settop(l, base); } - /* Close an unterminated transport group after its last retained record. */ - if (group && append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + /* + * A source group still open at the end continues in the next append: keep + * the output group open when it carries the source context, otherwise close + * it and reopen the source context for the records that follow. + */ + if (source_open && source_attrs && group && + (emitted_group_number != group_number || + !values_equal(group->cobj_record->variant, source_attrs->variant) || + !(has_otel_context(group->cobj_metadata) ? + context_metadata_equal(group->cobj_metadata->variant, source_meta->variant) : + values_equal(group->cobj_metadata->variant, source_meta->variant)))) { + if (append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + goto failure; + } + group = NULL; + } + + if (source_open && source_attrs) { + if (!group) { + if (reopen_source_group(ctx, out, source_attrs, source_meta, resource_id, + emitted_id, &output_bytes) != 0) { + goto failure; + } + } + } + else if (group && append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + /* Close an unterminated transport group after its last retained record. */ goto failure; } From 3c273ef3bafc7dcd23411db1f7ae2492d3132a32 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:35:30 -0600 Subject: [PATCH 35/61] tests: internal: fts: expect open input groups to stay open Signed-off-by: Eduardo Silva --- tests/internal/fts.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/internal/fts.c b/tests/internal/fts.c index 013f0d951bf..811f6116153 100644 --- a/tests/internal/fts.c +++ b/tests/internal/fts.c @@ -249,7 +249,8 @@ static void run_policy(const char *code, int grouped, const char *error_mode, } } } - TEST_CHECK(depth == 0); + /* a group left open by the input stays open for the next append */ + TEST_CHECK(depth == (grouped == 2 ? 1 : 0)); flb_log_event_decoder_destroy(&decoder); } TEST_CHECK(records == expected_records); From 5ab79297dc753ea4a620863c8d1e57be8b6a8371 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:03:51 -0600 Subject: [PATCH 36/61] processor_fts: compare duplicate context keys by position Context changes were detected by matching keys by name, so with duplicate keys an edit that set a key to its first value, or changed a later copy, looked unchanged and was discarded. Maps with duplicate keys are now compared by position; other maps keep the order independent comparison. Signed-off-by: Eduardo Silva --- plugins/processor_fts/fts_logs.c | 149 +++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/plugins/processor_fts/fts_logs.c b/plugins/processor_fts/fts_logs.c index 35e63223d95..62fbb7b4c3f 100644 --- a/plugins/processor_fts/fts_logs.c +++ b/plugins/processor_fts/fts_logs.c @@ -345,6 +345,124 @@ static int empty_context_map(struct cfl_variant *value, int kind) return FLB_FALSE; } +static int key_pair_compare(const void *a, const void *b) +{ + const struct cfl_kvpair *left = *(struct cfl_kvpair * const *) a; + const struct cfl_kvpair *right = *(struct cfl_kvpair * const *) b; + size_t left_length = cfl_sds_len(left->key); + size_t right_length = cfl_sds_len(right->key); + + if (left_length != right_length) { + return left_length < right_length ? -1 : 1; + } + + return memcmp(left->key, right->key, left_length); +} + +/* TRUE when a map repeats a key; sorting keeps adversarial maps O(n log n) */ +static int has_duplicate_keys(struct cfl_variant *value) +{ + int count; + int i; + int j; + int duplicate = FLB_FALSE; + struct cfl_list *head; + struct cfl_kvpair **pairs; + + if (!value || value->type != CFL_VARIANT_KVLIST) { + return FLB_FALSE; + } + + count = cfl_kvlist_count(value->data.as_kvlist); + if (count < 2) { + return FLB_FALSE; + } + + pairs = flb_malloc(sizeof(struct cfl_kvpair *) * count); + if (!pairs) { + flb_errno(); + /* without memory to check, compare strictly */ + return FLB_TRUE; + } + + i = 0; + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pairs[i++] = cfl_list_entry(head, struct cfl_kvpair, _head); + } + + if (count <= 8) { + for (i = 0; i < count && !duplicate; i++) { + for (j = i + 1; j < count; j++) { + if (key_pair_compare(&pairs[i], &pairs[j]) == 0) { + duplicate = FLB_TRUE; + break; + } + } + } + } + else { + qsort(pairs, count, sizeof(struct cfl_kvpair *), key_pair_compare); + for (i = 1; i < count; i++) { + if (key_pair_compare(&pairs[i - 1], &pairs[i]) == 0) { + duplicate = FLB_TRUE; + break; + } + } + } + + flb_free(pairs); + return duplicate; +} + +static int pair_key_equal(struct cfl_kvpair *left, struct cfl_kvpair *right) +{ + return cfl_sds_len(left->key) == cfl_sds_len(right->key) && + memcmp(left->key, right->key, cfl_sds_len(left->key)) == 0; +} + +static int empty_attributes_pair(struct cfl_kvpair *pair) +{ + return cfl_sds_len(pair->key) == 10 && memcmp(pair->key, "attributes", 10) == 0 && + empty_context_map(pair->val, FLB_FALSE); +} + +/* + * Positional comparison of a resource or scope map with duplicate keys, where + * matching by name would hide edits to later duplicates. Empty attributes + * placeholders are ignored on both sides. + */ +static int context_kind_equal_positional(struct cfl_variant *left, struct cfl_variant *right) +{ + struct cfl_list *a; + struct cfl_list *b; + struct cfl_list *a_end = &left->data.as_kvlist->list; + struct cfl_list *b_end = &right->data.as_kvlist->list; + struct cfl_kvpair *pa; + struct cfl_kvpair *pb; + + a = a_end->next; + b = b_end->next; + while (1) { + while (a != a_end && empty_attributes_pair(cfl_list_entry(a, struct cfl_kvpair, _head))) { + a = a->next; + } + while (b != b_end && empty_attributes_pair(cfl_list_entry(b, struct cfl_kvpair, _head))) { + b = b->next; + } + if (a == a_end || b == b_end) { + return a == a_end && b == b_end; + } + + pa = cfl_list_entry(a, struct cfl_kvpair, _head); + pb = cfl_list_entry(b, struct cfl_kvpair, _head); + if (!pair_key_equal(pa, pb) || !values_equal(pa->val, pb->val)) { + return FLB_FALSE; + } + a = a->next; + b = b->next; + } +} + static int context_kind_equal(struct cfl_variant *left, struct cfl_variant *right) { struct cfl_list *head; @@ -359,6 +477,10 @@ static int context_kind_equal(struct cfl_variant *left, struct cfl_variant *righ return values_equal(left, right); } + if (has_duplicate_keys(left) || has_duplicate_keys(right)) { + return context_kind_equal_positional(left, right); + } + cfl_list_foreach(head, &left->data.as_kvlist->list) { pair = cfl_list_entry(head, struct cfl_kvpair, _head); other = map_field(right, pair->key); @@ -386,7 +508,10 @@ static int context_kind_equal(struct cfl_variant *left, struct cfl_variant *righ static int context_attributes_equal(struct cfl_variant *left, struct cfl_variant *right) { struct cfl_list *head; + struct cfl_list *a; + struct cfl_list *b; struct cfl_kvpair *pair; + struct cfl_kvpair *pb; struct cfl_variant *other; if (!left || !right || @@ -394,6 +519,30 @@ static int context_attributes_equal(struct cfl_variant *left, struct cfl_variant return values_equal(left, right); } + /* duplicate keys are compared by position so later duplicates count */ + if (has_duplicate_keys(left) || has_duplicate_keys(right)) { + a = left->data.as_kvlist->list.next; + b = right->data.as_kvlist->list.next; + while (a != &left->data.as_kvlist->list && b != &right->data.as_kvlist->list) { + pair = cfl_list_entry(a, struct cfl_kvpair, _head); + pb = cfl_list_entry(b, struct cfl_kvpair, _head); + if (!pair_key_equal(pair, pb)) { + return FLB_FALSE; + } + if (strcmp(pair->key, "resource") == 0 || strcmp(pair->key, "scope") == 0) { + if (!context_kind_equal(pair->val, pb->val)) { + return FLB_FALSE; + } + } + else if (!values_equal(pair->val, pb->val)) { + return FLB_FALSE; + } + a = a->next; + b = b->next; + } + return a == &left->data.as_kvlist->list && b == &right->data.as_kvlist->list; + } + cfl_list_foreach(head, &left->data.as_kvlist->list) { pair = cfl_list_entry(head, struct cfl_kvpair, _head); other = map_field(right, pair->key); From 5c661b401efaf8a61c765653367da1e8fdf9e0bc Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:03:51 -0600 Subject: [PATCH 37/61] processor_fts: detect duplicate JSON keys by sorting fb.parse_json() checked duplicate keys with an unseeded hash table, which crafted colliding keys can make quadratic, without a bound when JIT disables the instruction budget. Sorting the keys is O(n log n) for any input and is charged to the instruction budget. Signed-off-by: Eduardo Silva --- plugins/processor_fts/fts_values.c | 101 +++++++++++++++++++---------- 1 file changed, 65 insertions(+), 36 deletions(-) diff --git a/plugins/processor_fts/fts_values.c b/plugins/processor_fts/fts_values.c index 9a112908087..77701883b1a 100644 --- a/plugins/processor_fts/fts_values.c +++ b/plugins/processor_fts/fts_values.c @@ -1682,10 +1682,71 @@ static const luaL_Reg value_methods[] = { {NULL, NULL} }; -struct json_key_slot { +static int json_key_compare(const void *a, const void *b) +{ + yyjson_val *left = *(yyjson_val * const *) a; + yyjson_val *right = *(yyjson_val * const *) b; + size_t left_length = yyjson_get_len(left); + size_t right_length = yyjson_get_len(right); + + if (left_length != right_length) { + return left_length < right_length ? -1 : 1; + } + + return memcmp(yyjson_get_str(left), yyjson_get_str(right), left_length); +} + +/* + * Duplicate keys are found by sorting the object keys: O(n log n) for any + * input, unlike hashing, which crafted colliding keys can make quadratic. + */ +static int json_object_has_duplicates(yyjson_val *value, struct fts_runtime *ctx) +{ + size_t i; + size_t index; + size_t count; yyjson_val *key; - uint64_t hash; -}; + yyjson_val *item; + yyjson_val **keys; + int duplicate = FLB_FALSE; + + count = yyjson_obj_size(value); + if (count < 2) { + return FLB_FALSE; + } + + /* sorting is native work, charge it to the instruction budget */ + if (!ctx->jit) { + if ((size_t) ctx->instructions_left <= count) { + ctx->instructions_left = 0; + return FLB_TRUE; + } + ctx->instructions_left -= count; + } + + keys = flb_malloc(sizeof(yyjson_val *) * count); + if (!keys) { + flb_errno(); + return FLB_TRUE; + } + + i = 0; + yyjson_obj_foreach(value, index, count, key, item) { + (void) item; + keys[i++] = key; + } + + qsort(keys, count, sizeof(yyjson_val *), json_key_compare); + for (i = 1; i < count; i++) { + if (json_key_compare(&keys[i - 1], &keys[i]) == 0) { + duplicate = FLB_TRUE; + break; + } + } + + flb_free(keys); + return duplicate; +} static size_t json_value_cost(yyjson_val *value, int depth) { @@ -1738,10 +1799,6 @@ static struct cfl_variant *json_value(yyjson_val *value, int depth, struct fts_r yyjson_val *item; size_t index; size_t count; - size_t capacity; - size_t slot; - uint64_t hash; - struct json_key_slot *keys; if (depth >= FTS_MAX_DEPTH || (!ctx->jit && --ctx->instructions_left <= 0)) { return NULL; @@ -1755,49 +1812,21 @@ static struct cfl_variant *json_value(yyjson_val *value, int depth, struct fts_r return NULL; } - count = yyjson_obj_size(value); - capacity = 1; - while (capacity < count * 2) { - capacity *= 2; - } - - keys = calloc(capacity, sizeof(*keys)); - if (keys == NULL) { - flb_errno(); + if (json_object_has_duplicates(value, ctx)) { cfl_variant_destroy(out); return NULL; } yyjson_obj_foreach(value, index, count, key, item) { - hash = cfl_hash_64bits(yyjson_get_str(key), yyjson_get_len(key)); - slot = hash & (capacity - 1); - while (keys[slot].key != NULL) { - if ((!ctx->jit && --ctx->instructions_left <= 0) || - (keys[slot].hash == hash && - yyjson_get_len(keys[slot].key) == yyjson_get_len(key) && - memcmp(yyjson_get_str(keys[slot].key), yyjson_get_str(key), - yyjson_get_len(key)) == 0)) { - free(keys); - cfl_variant_destroy(out); - return NULL; - } - slot = (slot + 1) & (capacity - 1); - } - - keys[slot].key = key; - keys[slot].hash = hash; - child = json_value(item, depth + 1, ctx); if (!child || cfl_kvlist_insert_s(map, (char *) yyjson_get_str(key), yyjson_get_len(key), child) != 0) { - free(keys); cfl_variant_destroy(child); cfl_variant_destroy(out); return NULL; } } - free(keys); return out; } From 83a09e699198a3f484123c2450fe5cf28c366b94 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:04:05 -0600 Subject: [PATCH 38/61] tests: internal: processor: cover per-run chunk codecs Signed-off-by: Eduardo Silva --- tests/internal/processor.c | 143 +++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/tests/internal/processor.c b/tests/internal/processor.c index fc260fe8cb8..0880d896af7 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -1167,6 +1168,147 @@ static void processor_log_windows_need_every_unit(void) window_config_destroy(config); } +/* + * The decoded chunk outlives the first unit's lock, so it must decode and + * encode with its own codec: another worker thread can reset the first + * unit's decoder while later units still read records from the chunk. + */ +static int codec_shared; + +static int probe_codec_and_stop(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_record *record; + + /* callbacks run under the unit lock, so the counter needs no atomics */ + if (chunk->log_decoder == ins->log_decoder || chunk->log_encoder == ins->log_encoder) { + codec_shared++; + } + + /* leave the remaining records for the next unit to decode */ + flb_mp_chunk_cobj_record_next(chunk, &record); + return FLB_PROCESSOR_SUCCESS; +} + +static struct flb_processor_plugin codec_probe_plugin = { + .name = "test_codec_probe", + .description = "checks the chunk codec and stops early", + .cb_process_logs = probe_codec_and_stop, + .flags = 0 +}; + +static struct flb_processor_plugin codec_window_probe_plugin = { + .name = "test_codec_window_probe", + .description = "checks the chunk codec and stops early, windowed", + .cb_process_logs = probe_codec_and_stop, + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS +}; + +struct codec_worker { + struct flb_processor *proc; + char *input; + size_t input_size; + int expected; + int failures; +}; + +static void *codec_worker_run(void *data) +{ + int i; + int ret; + int records; + void *output; + size_t output_size; + struct codec_worker *worker = data; + + for (i = 0; i < 50; i++) { + records = -1; + output = NULL; + ret = flb_processor_run_with_count(worker->proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + worker->input, worker->input_size, + &output, &output_size, -1, &records); + if (ret != 0 || records != worker->expected || output == NULL) { + worker->failures++; + } + if (output != NULL && output != worker->input) { + flb_free(output); + } + } + + return NULL; +} + +static void run_codec_workers(struct flb_config *config, char *first, char *second, + char *input, size_t input_size, int expected) +{ + int i; + struct flb_processor *proc; + struct flb_input_instance owner; + struct codec_worker workers[4]; + pthread_t threads[4]; + + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + proc = window_processor_create(config, &owner, first, second); + TEST_ASSERT(proc != NULL); + + for (i = 0; i < 4; i++) { + workers[i].proc = proc; + workers[i].input = input; + workers[i].input_size = input_size; + workers[i].expected = expected; + workers[i].failures = 0; + TEST_ASSERT(pthread_create(&threads[i], NULL, codec_worker_run, &workers[i]) == 0); + } + for (i = 0; i < 4; i++) { + pthread_join(threads[i], NULL); + TEST_CHECK(workers[i].failures == 0); + } + + flb_processor_destroy(proc); + cmt_destroy(owner.cmt); +} + +static void processor_chunk_codec_is_per_run(void) +{ + int i; + char *input; + size_t input_size; + struct flb_config *config; + struct flb_log_event_encoder *encoder; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + mk_list_add(&codec_probe_plugin._head, &config->processor_plugins); + mk_list_add(&codec_window_probe_plugin._head, &config->processor_plugins); + codec_shared = 0; + + /* single chunk path: the second unit decodes what the first one left */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (i = 0; i < 300; i++) { + TEST_ASSERT(append_window_record(encoder, i) == 0); + } + run_codec_workers(config, "test_codec_probe", "test_single_mark", + encoder->output_buffer, encoder->output_length, 300); + flb_log_event_encoder_destroy(encoder); + + /* windowed path */ + TEST_ASSERT(create_window_records(&input, &input_size) == 0); + run_codec_workers(config, "test_codec_window_probe", "test_window_mark", + input, input_size, WINDOW_TOTAL_RECORDS); + flb_free(input); + + TEST_CHECK(codec_shared == 0); + + mk_list_del(&codec_probe_plugin._head); + mk_list_del(&codec_window_probe_plugin._head); + window_config_destroy(config); +} + TEST_LIST = { {"processor_metrics_failure_clears_output", processor_metrics_failure_clears_output}, { "processor_private_inputs_use_main_loop", processor_private_inputs_use_main_loop }, @@ -1176,5 +1318,6 @@ TEST_LIST = { { "processor_log_windows_match_single_chunk", processor_log_windows_match_single_chunk }, { "processor_log_windows_drop_all", processor_log_windows_drop_all }, { "processor_log_windows_need_every_unit", processor_log_windows_need_every_unit }, + { "processor_chunk_codec_is_per_run", processor_chunk_codec_is_per_run }, { 0 } }; From f06af5ad9076ada6f0d58183b3188f96d23bf2a2 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:04:05 -0600 Subject: [PATCH 39/61] tests: internal: fts: cover split groups, duplicate context keys and JSON Signed-off-by: Eduardo Silva --- tests/internal/fts.c | 211 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 211 insertions(+) diff --git a/tests/internal/fts.c b/tests/internal/fts.c index 811f6116153..571f999529c 100644 --- a/tests/internal/fts.c +++ b/tests/internal/fts.c @@ -51,6 +51,7 @@ static void run_policy(const char *code, int grouped, const char *error_mode, int groups = 0; int ret; int depth = 0; + int dup_count; const char *kind; const char zero_id[16] = {0}; msgpack_object empty_map = {.type = MSGPACK_OBJECT_MAP, .via.map = {0, NULL}}; @@ -109,6 +110,12 @@ static void run_policy(const char *code, int grouped, const char *error_mode, FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&empty_map)); TEST_CHECK(ret == 0); } + if (grouped == 11) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("duplicate"), FLB_LOG_EVENT_CSTRING_VALUE("one"), + FLB_LOG_EVENT_CSTRING_VALUE("duplicate"), FLB_LOG_EVENT_CSTRING_VALUE("two")); + TEST_CHECK(ret == 0); + } ret = flb_log_event_encoder_append_body_values(encoder, FLB_LOG_EVENT_CSTRING_VALUE("producer"), FLB_LOG_EVENT_CSTRING_VALUE("original")); TEST_CHECK(ret == 0); @@ -185,6 +192,22 @@ static void run_policy(const char *code, int grouped, const char *error_mode, if (grouped == 6) { TEST_CHECK(field(event.body, "empty") == NULL); } + if (grouped == 11) { + /* the edit left a single copy of the duplicated key */ + dup_count = 0; + for (i = 0; i < (int) event.body->via.map.size; i++) { + value = &event.body->via.map.ptr[i].key; + if (value->via.str.size == 9 && + memcmp(value->via.str.ptr, "duplicate", 9) == 0) { + dup_count++; + value = &event.body->via.map.ptr[i].val; + TEST_CHECK(value->type == MSGPACK_OBJECT_STR && + value->via.str.size == 3 && + memcmp(value->via.str.ptr, "one", 3) == 0); + } + } + TEST_CHECK(dup_count == 1); + } if (grouped == 8) { TEST_CHECK(field(event.body, "resource") == NULL); TEST_CHECK(field(event.body, "scope") == NULL); @@ -1176,6 +1199,189 @@ static void empty_metadata_is_materialized_independently(void) "previous=log.metadata:copy() end", 0, "drop", 6, 0, 0, NULL, NULL); } +/* Group layouts split across appends: 'S' starts a group, 'E' ends one */ +static void append_split_group(struct flb_log_event_encoder *encoder, const char *name) +{ + int ret; + + TEST_CHECK(flb_log_event_encoder_group_init(encoder) == 0); + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("batch"), FLB_LOG_EVENT_CSTRING_VALUE(name), + FLB_LOG_EVENT_CSTRING_VALUE("schema"), FLB_LOG_EVENT_BOOLEAN_VALUE(FLB_FALSE)); + TEST_CHECK(ret == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("producer"), FLB_LOG_EVENT_CSTRING_VALUE(name)); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_group_header_end(encoder) == 0); +} + +static void append_split_record(struct flb_log_event_encoder *encoder, int index) +{ + int ret; + struct flb_time ts; + + TEST_CHECK(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(&ts, 1700000000 + index, 0); + TEST_CHECK(flb_log_event_encoder_set_timestamp(encoder, &ts) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("index"), FLB_LOG_EVENT_INT64_VALUE(index)); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_commit_record(encoder) == 0); +} + +/* layout tokens: r record, S group start, E group end */ +static void run_split_layout(const char *code, const char *error_mode, + const char *layout, const char *expected) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + msgpack_object *value; + void *output = NULL; + size_t output_size = 0; + char result[256]; + char token[32]; + const char *cursor; + int type; + int length; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "split_groups", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", code) == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "error_mode", error_mode) == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (cursor = layout; *cursor; ) { + if (sscanf(cursor, "%31s%n", token, &length) != 1) { + break; + } + cursor += length; + if (token[0] == 'r') { + append_split_record(encoder, atoi(token + 1)); + } + else if (token[0] == 'S') { + append_split_group(encoder, token + 1); + } + else if (token[0] == 'E') { + TEST_CHECK(flb_log_event_encoder_group_end(encoder) == 0); + } + } + + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, + &output, &output_size) == 0); + + result[0] = '\0'; + if (output_size > 0) { + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type == FLB_LOG_EVENT_GROUP_START) { + value = field(event.body, "producer"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_STR); + snprintf(token, sizeof(token), "S%.*s ", (int) value->via.str.size, + value->via.str.ptr); + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + snprintf(token, sizeof(token), "E "); + } + else { + value = field(event.body, "index"); + TEST_ASSERT(value != NULL); + snprintf(token, sizeof(token), "r%d ", (int) value->via.i64); + } + strncat(result, token, sizeof(result) - strlen(result) - 1); + } + flb_log_event_decoder_destroy(&decoder); + } + if (strlen(result) > 0) { + result[strlen(result) - 1] = '\0'; + } + TEST_CHECK(strcmp(result, expected) == 0); + TEST_MSG("layout '%s' produced '%s', expected '%s'", layout, result, expected); + + if (output && output != encoder->output_buffer) { + flb_free(output); + } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void groups_split_across_appends_are_preserved(void) +{ + const char *noop = "function process(log) end"; + + /* records continuing a group from a previous append, then their end */ + run_split_layout(noop, "drop", "r0 r1 E Sg r2 E", "r0 r1 E Sg r2 E"); + /* a group left open for the next append */ + run_split_layout(noop, "drop", "Sg r0 r1", "Sg r0 r1"); + run_split_layout(noop, "drop", "r0 E Sg r1 E Sh r2", "r0 E Sg r1 E Sh r2"); + /* an open group whose records are all dropped still opens for the next append */ + run_split_layout("local fb=require('fluentbit'); " + "function process(log) if log.body:get_number('index') > 0 " + "then return fb.DROP end end", + "drop", "r0 E Sg r1 r2", "r0 E Sg"); + /* only the first stray end before any start belongs to a previous append */ + run_split_layout(noop, "drop", "r0 E E Sg r1 E E r2", "r0 E Sg r1 E r2"); + /* + * a nested start begins a new flat context; nested open groups are not a + * single group continued by the next append, so the open one is dropped + */ + run_split_layout(noop, "drop", "Sg r0 Sh r1", "Sg r0 E r1"); +} + +static void open_group_edits_reopen_the_source_context(void) +{ + /* the last record moves to an edited context, the source group stays open */ + run_split_layout("function process(log) if log.body:get_number('index') == 1 then " + "log.context.attributes:set('producer', 'edited') end end", + "drop", "Sg r0 r1", "Sg r0 E Sedited r1 E Sg"); +} + +static void continuation_records_cannot_set_a_context(void) +{ + const char *code = "function process(log) if log.body:get_number('index') == 0 then " + "log.context.attributes:set('producer', 'new') end end"; + + run_split_layout(code, "keep", "r0 r1 E Sg r2 E", "r0 r1 E Sg r2 E"); + run_split_layout(code, "drop", "r0 r1 E Sg r2 E", "r1 E Sg r2 E"); +} + +static void duplicate_context_keys_keep_edits(void) +{ + /* + * 'duplicate' holds 'one' and 'two': setting it to the first value must + * still remove the second copy instead of looking unchanged. + */ + run_policy("function process(log) log.context.attributes:set('duplicate', 'one'); " + "assert(log.context.attributes:get_string('duplicate') == 'one') end", + 11, "drop", 6, 2, 0, NULL, NULL); +} + +static void json_large_objects_parse_in_linear_time(void) +{ + /* 60000 distinct keys under the default instruction budget, then a duplicate */ + run_policy("local fb=require('fluentbit'); local parts={}; " + "for i=1,60000 do parts[i]=string.format('\"k%06d\":0', i) end; " + "local text='{'..table.concat(parts, ',')..'}'; " + "local dup='{'..table.concat(parts, ',')..',\"k000001\":1}'; " + "function process(log) " + "assert(fb.parse_json(text):len() == 60000); " + "assert(not pcall(fb.parse_json, dup)) end", + 0, "drop", 6, 0, 0, "memory_limit", "128M"); +} + TEST_LIST = { {"empty_metadata_cow_isolation", empty_metadata_is_materialized_independently}, {"edited_record_one_policy_tree", edited_record_needs_one_policy_tree}, @@ -1229,5 +1435,10 @@ TEST_LIST = { {"context_replacement_during_collection", context_replacement_during_collection}, {"deep_mutable_views", deep_mutable_views}, {"garbage_collector_is_private", garbage_collector_is_private}, + {"groups_split_across_appends", groups_split_across_appends_are_preserved}, + {"open_group_edits_reopen_source", open_group_edits_reopen_the_source_context}, + {"continuation_records_no_context", continuation_records_cannot_set_a_context}, + {"duplicate_context_keys_keep_edits", duplicate_context_keys_keep_edits}, + {"json_large_objects_linear", json_large_objects_parse_in_linear_time}, {NULL, NULL} }; From 407ac5e9701aac9dc599e250b6990357733d74b1 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:04:05 -0600 Subject: [PATCH 40/61] tests: integration: cover groups split across appends Signed-off-by: Eduardo Silva --- .../processor_fts/tests/test_fts_records.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_records.py b/tests/integration/scenarios/processor_fts/tests/test_fts_records.py index da7b2af6fe9..5e03623b347 100644 --- a/tests/integration/scenarios/processor_fts/tests/test_fts_records.py +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_records.py @@ -522,3 +522,99 @@ def test_missing_transport_ids_are_group_local(tmp_path, error_mode): assert "cannot construct policy output" not in log_text(service) finally: service.stop() + + +def start_otlp_json_service(tmp_path, processors): + source = {"name": "forward", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", "tag": "stream"} + if processors: + source["processors"] = {"logs": processors} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + config = tmp_path / "otlp-json.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 1, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + service.start() + return service + + +def otlp_resources(service): + """(resource attributes, record bodies) for every exported resource.""" + resources = [] + for line in log_text(service).splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + for resource in json.loads(line)["resourceLogs"]: + attrs = [(item["key"], item["value"]) for item in + resource.get("resource", {}).get("attributes", [])] + bodies = [{item["key"]: item["value"] for item in log["body"]["kvlistValue"]["values"]} + for scope in resource["scopeLogs"] for log in scope["logRecords"]] + resources.append((attrs, bodies)) + return resources + + +def raw_map(pairs): + """Map with explicit, possibly duplicated, keys.""" + return Raw(b"\xdf" + len(pairs).to_bytes(4, "big") + + b"".join(pack(key) + pack(value) for key, value in pairs)) + + +@pytest.mark.parametrize("chain", ["fts", "content_modifier", "fts_and_content_modifier"]) +def test_groups_split_across_appends_keep_their_records(tmp_path, chain): + """A group whose end arrives in a later append keeps every record.""" + fts = {"name": "fts", "code": "function process(log) log.body:set('edited', true) end"} + modifier = {"name": "content_modifier", "action": "insert", "key": "native", "value": "v"} + processors = {"fts": [fts], "content_modifier": [modifier], + "fts_and_content_modifier": [fts, modifier]}[chain] + service = start_otlp_json_service(tmp_path, processors) + try: + start, end = event_time(0xFFFFFFFF), event_time(0xFFFFFFFE) + now = event_time(1700000000) + head = [[[start, {"schema": "otlp", "resource_id": 0, "scope_id": 0}], + {"resource": {"attributes": {"service.name": "split"}}, "scope": {"name": "s"}}], + [[now, {"otlp": {}}], {"msg": "r1"}]] + tail = [[[now, {"otlp": {}}], {"msg": "r2"}], [[end, {}], {}]] + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as connection: + connection.sendall(pack(["stream", head])) + time.sleep(0.1) + connection.sendall(pack(["stream", tail])) + resources = service.wait_for_condition( + lambda: otlp_resources(service) if sum(len(b) for _, b in otlp_resources(service)) >= 2 + else None, timeout=20, description="two exported records") + # both records stay in the resource opened by the first append + assert len(resources) == 1 + attrs, bodies = resources[0] + assert attrs == [("service.name", {"stringValue": "split"})] + assert [body["msg"]["stringValue"] for body in bodies] == ["r1", "r2"] + for body in bodies: + if "fts" in chain: + assert body["edited"] == {"boolValue": True} + if "content_modifier" in chain: + assert body["native"] == {"stringValue": "v"} + assert "cannot construct policy output" not in log_text(service) + finally: + service.stop() + + +def test_duplicate_resource_attributes_collapse_on_edit(tmp_path): + """Setting a duplicated resource attribute exports a single value.""" + fts = {"name": "fts", "code": '''function process(log) + log.context.attributes:get("resource"):get("attributes"):set("service.name", "a") + end'''} + service = start_otlp_json_service(tmp_path, [fts]) + try: + start, end = event_time(0xFFFFFFFF), event_time(0xFFFFFFFE) + now = event_time(1700000000) + attributes = raw_map([("service.name", "a"), ("service.name", "b")]) + entries = [[[start, {"schema": "otlp", "resource_id": 0, "scope_id": 0}], + {"resource": {"attributes": attributes}, "scope": {"name": "s"}}], + [[now, {"otlp": {}}], {"msg": "x"}], + [[end, {}], {}]] + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as connection: + connection.sendall(pack(["stream", entries])) + resources = service.wait_for_condition(lambda: otlp_resources(service), timeout=20, + description="exported resource") + assert resources[0][0] == [("service.name", {"stringValue": "a"})] + assert "policy failed" not in log_text(service) + finally: + service.stop() From 6f4970ef38f1d95248af3baa59f0d292138b1b68 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:22:54 -0600 Subject: [PATCH 41/61] lib: cmetrics: keep label attributes for unrepresentable typed values Typed data point attributes with no OTLP form, such as null values or unsigned integers above INT64_MAX, made the conversion return NULL, which was reported as an allocation error and dropped the whole metric. Check the values first and keep the label derived attributes for them, so a NULL conversion result is again only an allocation failure. Signed-off-by: Eduardo Silva --- lib/cmetrics/src/cmt_encode_opentelemetry.c | 66 +++++++++++++++++++++ lib/cmetrics/tests/opentelemetry.c | 39 ++++++++++++ 2 files changed, 105 insertions(+) diff --git a/lib/cmetrics/src/cmt_encode_opentelemetry.c b/lib/cmetrics/src/cmt_encode_opentelemetry.c index 91dd4ac7999..f384e13c993 100644 --- a/lib/cmetrics/src/cmt_encode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_encode_opentelemetry.c @@ -1316,6 +1316,67 @@ static void attribute_arena_free(void *data, void *pointer) /* The encoder releases all datapoint objects with their arena. */ } +/* + * TRUE when every value can be represented as an OTLP AnyValue: NULL values, + * unsigned integers above INT64_MAX and unknown types have no OTLP form. + */ +static int otlp_variant_is_representable(struct cfl_variant *value, int depth) +{ + size_t index; + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_array *array; + + if (value == NULL || depth > 64) { + return CMT_FALSE; + } + + switch (value->type) { + case CFL_VARIANT_STRING: + case CFL_VARIANT_BOOL: + case CFL_VARIANT_INT: + case CFL_VARIANT_DOUBLE: + case CFL_VARIANT_BYTES: + case CFL_VARIANT_REFERENCE: + return CMT_TRUE; + case CFL_VARIANT_UINT: + return value->data.as_uint64 <= INT64_MAX; + case CFL_VARIANT_ARRAY: + array = value->data.as_array; + for (index = 0; index < array->entry_count; index++) { + if (!otlp_variant_is_representable(array->entries[index], depth + 1)) { + return CMT_FALSE; + } + } + return CMT_TRUE; + case CFL_VARIANT_KVLIST: + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (!otlp_variant_is_representable(pair->val, depth + 1)) { + return CMT_FALSE; + } + } + return CMT_TRUE; + default: + return CMT_FALSE; + } +} + +static int otlp_kvlist_is_representable(struct cfl_kvlist *kvlist) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (!otlp_variant_is_representable(pair->val, 0)) { + return CMT_FALSE; + } + } + + return CMT_TRUE; +} + static int apply_typed_point_attributes(struct cmt_opentelemetry_context *context, struct cmt_map *map, struct cmt_metric *sample, void *data_point) @@ -1341,6 +1402,11 @@ static int apply_typed_point_attributes(struct cmt_opentelemetry_context *contex if (attributes == NULL || !cmt_otlp_attributes_match_labels(map, sample)) { return CMT_ENCODE_OPENTELEMETRY_SUCCESS; } + + /* typed values without an OTLP form keep the label derived attributes */ + if (!otlp_kvlist_is_representable(attributes)) { + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; + } count = cfl_kvlist_count(attributes); static_count = cfl_list_size(&context->cmt->static_labels->list); heap = cfl_kvlist_to_otlp_kvpair_list(attributes); diff --git a/lib/cmetrics/tests/opentelemetry.c b/lib/cmetrics/tests/opentelemetry.c index 996df4049f9..99dfc35fada 100644 --- a/lib/cmetrics/tests/opentelemetry.c +++ b/lib/cmetrics/tests/opentelemetry.c @@ -2851,6 +2851,44 @@ static void test_otlp_unsigned_attribute_preserves_signed_range(void) cmt_destroy(ctx); } +/* Typed attributes without an OTLP form keep the label derived attributes. */ +static void test_otlp_unrepresentable_attribute_keeps_point(void) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cfl_kvlist *attrs; + cfl_sds_t wire; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "huge_attribute", "huge attribute", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + gauge->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->metric.otlp_metadata != NULL); + attrs = cfl_kvlist_create(); + TEST_ASSERT(attrs != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(attrs, "huge", UINT64_MAX) == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->metric.otlp_metadata, "attributes", attrs) == 0); + TEST_ASSERT(cmt_otlp_capture_labels(gauge->map, &gauge->map->metric) == 0); + + wire = cmt_encode_opentelemetry_create(ctx); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + TEST_ASSERT(request->n_resource_metrics == 1); + metric = request->resource_metrics[0]->scope_metrics[0]->metrics[0]; + TEST_CHECK(strcmp(metric->name, "huge_attribute") == 0); + TEST_ASSERT(metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->as_double == 42); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_destroy(ctx); +} + /* Repeated label-less points of one metric are the same series: the last one * replaces the static sample, as released versions behave. */ static cfl_sds_t generate_repeated_unlabeled_points_payload() @@ -3017,6 +3055,7 @@ static void test_opentelemetry_repeated_unlabeled_points_last_wins(void) } TEST_LIST = { + {"otlp_unrepresentable_attribute_keeps_point", test_otlp_unrepresentable_attribute_keeps_point}, {"repeated_unlabeled_points_last_wins", test_opentelemetry_repeated_unlabeled_points_last_wins}, {"otlp_unsigned_attribute_signed_range", test_otlp_unsigned_attribute_preserves_signed_range}, {"opentelemetry_api_full_roundtrip_with_msgpack", test_opentelemetry_api_full_roundtrip_with_msgpack}, From ce21873eafb83d39a1e5d0181eb0e91e028e5726 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:22:54 -0600 Subject: [PATCH 42/61] processor_fts: copy protobuf keys with the system allocator Messages built for typed telemetry are released by protobuf-c with the system allocator, but attribute keys were copied with flb_strdup(), which uses the embedded jemalloc when it is enabled. Signed-off-by: Eduardo Silva --- plugins/processor_fts/fts_proto.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/plugins/processor_fts/fts_proto.c b/plugins/processor_fts/fts_proto.c index dfcb1b77ed0..4bcd7c52344 100644 --- a/plugins/processor_fts/fts_proto.c +++ b/plugins/processor_fts/fts_proto.c @@ -466,14 +466,20 @@ static int write_repeated(const ProtobufCFieldDescriptor *field, struct cfl_vari return -1; } - entry->key = flb_strdup(pair->key); - entry->value = (void *) fts_proto_build( - &opentelemetry__proto__common__v1__any_value__descriptor, - pair->val, budget, depth + 1); + /* + * protobuf-c releases the message with the system allocator, so + * the key is copied with malloc() like every other string. + */ + entry->key = malloc(cfl_sds_len(pair->key) + 1); if (!entry->key) { flb_errno(); return -1; } + memcpy(entry->key, pair->key, cfl_sds_len(pair->key) + 1); + + entry->value = (void *) fts_proto_build( + &opentelemetry__proto__common__v1__any_value__descriptor, + pair->val, budget, depth + 1); if (!entry->value) { return -1; From 46f8d9011e21d61a18528f1196ca51c49082173d Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:22:54 -0600 Subject: [PATCH 43/61] output: use the serialized size for processed profiles Processed profiles were exposed with the buffer capacity, which can exceed the serialized bytes and append zeroed memory to the event chunk, as already fixed for traces. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_output.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index 7f496e8a3c2..9ce92bdc388 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -1253,6 +1253,8 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, return NULL; } + /* Capacity can exceed the serialized profile bytes after processing. */ + p_size = serialization_buffer_offset; out_flush->processed_event_chunk = flb_event_chunk_create( evc->type, 0, From d431238731c3b6c6f1cdb54b5e459f145ea5fa9e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:22:54 -0600 Subject: [PATCH 44/61] tests: internal: mp: keep corrupted msgpack counts small Corrupting the count of a 16 or 32 bit array or map header made the msgpack-c fallback request gigabytes, which fails under AddressSanitizer. Signed-off-by: Eduardo Silva --- tests/internal/mp.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/internal/mp.c b/tests/internal/mp.c index bc43203944b..891cfc3eabb 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -805,6 +805,17 @@ static void test_log_scan_matches_decoder(void) TEST_CHECK(flb_mp_count_log_records(encoder->output_buffer, encoder->output_length) == reference_log_count(encoder->output_buffer, encoder->output_length)); encoder->output_buffer[index] = saved; + + /* + * Leave the counts of 16 and 32 bit array and map headers intact: a + * corrupted count makes the msgpack-c fallback request gigabytes. + */ + if ((unsigned char) saved == 0xdc || (unsigned char) saved == 0xde) { + index += 2; + } + else if ((unsigned char) saved == 0xdd || (unsigned char) saved == 0xdf) { + index += 4; + } } flb_log_event_encoder_destroy(encoder); } From c623a95f58592c231a8eb210137968567bc883bb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:22:54 -0600 Subject: [PATCH 45/61] tests: internal: opentelemetry: fix literal length and Windows build Derive the appended literal length instead of hardcoding it, which read past the literal, and use the Windows descriptor functions to capture stderr. Signed-off-by: Eduardo Silva --- tests/internal/opentelemetry.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 0c44e6e318e..128aea0e7d0 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -46,7 +46,19 @@ #include #include #include +#ifdef _WIN32 +#include +#define test_dup(fd) _dup(fd) +#define test_dup2(fd, target) _dup2(fd, target) +#define test_close(fd) _close(fd) +#define test_fileno(stream) _fileno(stream) +#else #include +#define test_dup(fd) dup(fd) +#define test_dup2(fd, target) dup2(fd, target) +#define test_close(fd) close(fd) +#define test_fileno(stream) fileno(stream) +#endif #include #include @@ -3821,7 +3833,8 @@ void test_json_invalid_attributes_warn_once() for (index = 0; index < 50; index++) { flb_sds_printf(&json, "{\"key\":\"b%d\",\"value\":{\"bytesValue\":\"!!!\"}},", index); } - flb_sds_cat_safe(&json, "{\"key\":\"ok\",\"value\":{\"stringValue\":\"kept\"}}]", 46); + flb_sds_cat_safe(&json, "{\"key\":\"ok\",\"value\":{\"stringValue\":\"kept\"}}]", + sizeof("{\"key\":\"ok\",\"value\":{\"stringValue\":\"kept\"}}]") - 1); ret = flb_pack_json(json, flb_sds_len(json), &buffer, &size, &type, NULL); TEST_ASSERT(ret == 0); @@ -3837,15 +3850,15 @@ void test_json_invalid_attributes_warn_once() TEST_ASSERT(capture != NULL); fflush(stderr); - saved_stderr = dup(fileno(stderr)); + saved_stderr = test_dup(test_fileno(stderr)); TEST_ASSERT(saved_stderr >= 0); - dup2(fileno(capture), fileno(stderr)); + test_dup2(test_fileno(capture), test_fileno(stderr)); ret = flb_otel_utils_clone_kvlist_from_otlp_json_array(attributes, &unpacked.data); fflush(stderr); - dup2(saved_stderr, fileno(stderr)); - close(saved_stderr); + test_dup2(saved_stderr, test_fileno(stderr)); + test_close(saved_stderr); TEST_CHECK(ret == 0); TEST_CHECK(cfl_kvlist_count(attributes) == 1); From 447539cb2ad2616f654b1b539c93267aa5eb75c2 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 22:30:59 -0600 Subject: [PATCH 46/61] processor_fts: clear stale stack poisoning after Lua errors in ASan builds LuaJIT raises errors by resetting the C stack pointer instead of calling longjmp(), so AddressSanitizer keeps the redzones of the skipped frames poisoned and later calls report bogus stack errors, as seen with the gcc 9 and clang 12 sanitizer jobs. After a failed protected call, unpoison the dead stack below the current frame. Builds without the sanitizer are not affected. Signed-off-by: Eduardo Silva --- plugins/processor_fts/fts.h | 1 + plugins/processor_fts/fts_runtime.c | 41 ++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/plugins/processor_fts/fts.h b/plugins/processor_fts/fts.h index f3112ec4e1d..86cc2d7deae 100644 --- a/plugins/processor_fts/fts.h +++ b/plugins/processor_fts/fts.h @@ -116,6 +116,7 @@ extern const struct fts_signal_adapter fts_logs_adapter; int fts_runtime_initialize(struct fts_runtime *ctx, struct flb_config *config); void fts_runtime_destroy(struct fts_runtime *ctx); void fts_report_policy_error(struct flb_processor_instance *ins); +void fts_sanitizer_forget_unwound_frames(void); int fts_runtime_call(struct fts_runtime *ctx, void *data, int results); int fts_register_values(lua_State *l); void fts_collect(struct fts_runtime *ctx, int emergency); diff --git a/plugins/processor_fts/fts_runtime.c b/plugins/processor_fts/fts_runtime.c index b0bdd69f44c..68222f75b5e 100644 --- a/plugins/processor_fts/fts_runtime.c +++ b/plugins/processor_fts/fts_runtime.c @@ -24,8 +24,36 @@ #include #include +#ifdef FLB_HAVE_SANITIZE_ADDRESS +#include +#endif + #include "fts.h" +/* + * LuaJIT raises errors by resetting the C stack pointer instead of calling + * longjmp(), so AddressSanitizer never sees the frames it skips and keeps + * their redzones poisoned. Later calls reusing that stack report bogus + * errors. After a failed protected call, clear the shadow of the stack below + * the current frame, which only holds those dead frames; the window covers + * nested C and Lua calls up to LuaJIT's C call limit. Builds without the + * sanitizer compile this to nothing. + */ +#define FTS_UNWOUND_STACK_SIZE (256 * 1024) + +#ifdef FLB_HAVE_SANITIZE_ADDRESS +__attribute__((noinline)) +#endif +void fts_sanitizer_forget_unwound_frames(void) +{ +#ifdef FLB_HAVE_SANITIZE_ADDRESS + char *top; + + top = (char *) __builtin_frame_address(0); + __asan_unpoison_memory_region(top - FTS_UNWOUND_STACK_SIZE, FTS_UNWOUND_STACK_SIZE); +#endif +} + /* A policy cannot catch budget exhaustion and continue an endless loop. */ static int policy_pcall(lua_State *l) { @@ -35,6 +63,9 @@ static int policy_pcall(lua_State *l) luaL_checktype(l, 1, LUA_TFUNCTION); ret = lua_pcall(l, lua_gettop(l) - 1, LUA_MULTRET, 0); + if (ret != 0) { + fts_sanitizer_forget_unwound_frames(); + } lua_getallocf(l, &data); ctx = data; @@ -180,7 +211,11 @@ int fts_load_helpers(lua_State *l, const char *code, size_t size, const char *na lua_rawgeti(l, LUA_REGISTRYINDEX, ctx->helpers_env_ref); lua_setfenv(l, -2); - return lua_pcall(l, 0, 0, 0); + if (lua_pcall(l, 0, 0, 0) != 0) { + fts_sanitizer_forget_unwound_frames(); + return -1; + } + return 0; } static int initialize_policy(lua_State *l) @@ -322,6 +357,7 @@ int fts_runtime_initialize(struct fts_runtime *ctx, struct flb_config *config) lua_pushlightuserdata(l, ctx); ret = lua_pcall(l, 1, 0, 0); if (ret != 0) { + fts_sanitizer_forget_unwound_frames(); return FLB_PROCESSOR_FAILURE; } @@ -368,6 +404,9 @@ int fts_runtime_call(struct fts_runtime *ctx, void *data, int results) lua_rawgeti(l, LUA_REGISTRYINDEX, ctx->callback_ref); lua_pushlightuserdata(l, data); ret = lua_pcall(l, 1, results, 0); + if (ret != 0) { + fts_sanitizer_forget_unwound_frames(); + } if (ret == 0 && !ctx->jit && ctx->instructions_left <= 0) { lua_settop(l, base); lua_pushliteral(l, "Lua instruction limit exceeded"); From fc74ecef8f2d25762dcd2787a76e1ab1586f1632 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:00 -0600 Subject: [PATCH 47/61] output: keep the group boundaries of stored chunks Output processors normalize their result. Keep the group markers that span chunks only when the stored chunk itself had them, like outputs without processors receive them. Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_output.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index 9ce92bdc388..abaf431a68c 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -902,9 +902,10 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, normalized_buf = NULL; normalized_size = 0; - ret = flb_mp_normalize_log_buffer_groups_msgpack(p_buf, p_size, - &normalized_buf, - &normalized_size); + /* keep only the group markers that span chunks in the stored data */ + ret = flb_mp_normalize_log_buffer_groups_msgpack_ex( + p_buf, p_size, &normalized_buf, &normalized_size, + flb_mp_log_group_boundaries(evc->data, evc->size)); if (ret == 0) { if (p_buf != evc->data) { flb_free(p_buf); From 2d37e7ee89177dda3800b9d1bc48427f655869b1 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:00 -0600 Subject: [PATCH 48/61] processor_fts: fail kept families whose name the policy output uses Native uint64 families that a policy cannot process are copied back with cmt_cat in keep mode, which merges them into a policy output family of the same name and gives the merged family their routing. Treat that collision as a batch failure, so keep forwards the original families. Signed-off-by: Eduardo Silva --- plugins/processor_fts/fts_metrics.c | 43 +++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/plugins/processor_fts/fts_metrics.c b/plugins/processor_fts/fts_metrics.c index 0b46e19abd6..eeff0cbd919 100644 --- a/plugins/processor_fts/fts_metrics.c +++ b/plugins/processor_fts/fts_metrics.c @@ -391,6 +391,40 @@ static int prepare_rejected_families(struct cmt *out, struct cmt *input, return 0; } +static int family_name_used(struct cmt *out, int type, const char *fqname) +{ + struct cfl_list *list; + struct cfl_list *head; + struct cmt_map *map; + + if (type == CMT_COUNTER) { + list = &out->counters; + } + else if (type == CMT_GAUGE) { + list = &out->gauges; + } + else { + list = &out->untypeds; + } + + cfl_list_foreach(head, list) { + if (type == CMT_COUNTER) { + map = cfl_list_entry(head, struct cmt_counter, _head)->map; + } + else if (type == CMT_GAUGE) { + map = cfl_list_entry(head, struct cmt_gauge, _head)->map; + } + else { + map = cfl_list_entry(head, struct cmt_untyped, _head)->map; + } + if (strcmp(map->opts->fqname, fqname) == 0) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + /* The input context stays owned by its producer, so kept families are copied. */ static int copy_rejected_families(struct cmt *out, struct unsupported_family *families, size_t count) @@ -405,6 +439,15 @@ static int copy_rejected_families(struct cmt *out, struct unsupported_family *fa struct cmt_untyped *untyped; for (index = 0; index < count; index++) { + /* + * cmt_cat merges into an existing family with the same name: a policy + * output family must never absorb the kept samples and their routing. + */ + if (family_name_used(out, families[index].type, + families[index].map->opts->fqname)) { + return -1; + } + if (families[index].type == CMT_COUNTER) { ret = cmt_cat_counter(out, families[index].family, NULL); list = &out->counters; From 99c7a310df5feede33c1056a830ec393453ccc7b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:00 -0600 Subject: [PATCH 49/61] processor_fts: document telemetry value lifetime and output limits Signed-off-by: Eduardo Silva --- plugins/processor_fts/README.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/plugins/processor_fts/README.md b/plugins/processor_fts/README.md index 7d548fb9a56..33429afc313 100644 --- a/plugins/processor_fts/README.md +++ b/plugins/processor_fts/README.md @@ -75,8 +75,13 @@ Size suffixes are decimal: `64K` is 64000 bytes. Changes are transactional per item. Edits to the body, metadata, timestamp, resource, and scope are committed or rolled back together. -If a processed batch exceeds `output_limit`, the whole batch is treated as -failed: `drop` discards it and `keep` forwards the original batch. +For logs, if a processed batch exceeds `output_limit`, the whole batch is +treated as failed: `drop` discards it and `keep` forwards the original batch. +For metrics and traces the limit is checked per item: an item whose result does +not fit follows `error_mode` (`drop` removes it, `keep` forwards the original +item), and the whole batch fails only when the kept originals do not fit +either. Internal failures, such as a batch that cannot be decoded, apply +`error_mode` to the whole batch. A batch is the set of records handed to one callback run. Small chunks are processed as one batch. Larger buffers, such as complete chunks reaching output @@ -136,10 +141,12 @@ and `fb.null`. ### Value lifetime -Values passed to a callback, and the views obtained from them, are valid only -during that callback. Using one later raises an error. To keep data between -callbacks, store `v:copy()`. Copies are independent: changing them never -changes a committed record. +Log values passed to a callback, and the views obtained from them, are valid +only during that callback. Using one later raises an error. Metric and span +values are snapshots made for the callback: they stay readable afterwards, but +changing them never affects later items or committed output, and keeping them +retains their memory. To keep data between callbacks, store `v:copy()`. Copies +are independent: changing them never changes a committed record. Lua globals persist for the lifetime of the processor instance. From faa7d40646644da1d7467eb26221c7c966c6e1fd Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:00 -0600 Subject: [PATCH 50/61] lib: cmetrics: keep OTLP metadata readable by older versions - Write the OTLP point and family metadata to the external otlp tree as well, keyed by series hash and timestamp, so Fluent Bit 5.1 readers keep start times, flags and exemplars. Readers of the sidecar drop that copy. - Copy metadata with a recursive copy instead of a msgpack round trip, which failed for values nested deeper than the msgpack limit. - Skip only the sidecar entries that cannot be read back. - Append allow_reset to the decode context to keep its field layout. Signed-off-by: Eduardo Silva --- .../include/cmetrics/cmt_decode_msgpack.h | 3 +- .../include/cmetrics/cmt_otlp_metadata.h | 1 + lib/cmetrics/src/cmt_encode_msgpack.c | 2 +- lib/cmetrics/src/cmt_otlp_metadata.c | 342 ++++++++++++++++-- lib/cmetrics/tests/opentelemetry.c | 168 +++++++++ 5 files changed, 491 insertions(+), 25 deletions(-) diff --git a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h index aed6097b2ad..4bffc827c3f 100644 --- a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h +++ b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h @@ -55,9 +55,10 @@ struct cmt_msgpack_decode_context { uint64_t *summary_quantiles; size_t summary_quantiles_count; int aggregation_type; - int allow_reset; int metric_value_type_set; int metric_section_set; + /* Appended to keep the established field layout. */ + int allow_reset; }; int cmt_decode_msgpack_create(struct cmt **out_cmt, char *in_buf, size_t in_size, diff --git a/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h b/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h index 0c6aa0472e3..fef8eb3c57a 100644 --- a/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h +++ b/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h @@ -26,6 +26,7 @@ struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source); int cmt_otlp_metadata_pack(mpack_writer_t *writer, struct cmt *context); +int cmt_otlp_metadata_pack_external(mpack_writer_t *writer, struct cmt *context); int cmt_otlp_metadata_restore(struct cmt *context); int cmt_otlp_histogram_restore(struct cmt *context); int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample); diff --git a/lib/cmetrics/src/cmt_encode_msgpack.c b/lib/cmetrics/src/cmt_encode_msgpack.c index bdf9ffe7ebf..a9b2555f80f 100644 --- a/lib/cmetrics/src/cmt_encode_msgpack.c +++ b/lib/cmetrics/src/cmt_encode_msgpack.c @@ -486,7 +486,7 @@ static int pack_context_header(mpack_writer_t *writer, struct cmt *cmt) } mpack_write_cstr(writer, "external"); - result = pack_cfl_variant_kvlist(writer, cmt->external_metadata); + result = cmt_otlp_metadata_pack_external(writer, cmt); if (result != 0) { return -2; diff --git a/lib/cmetrics/src/cmt_otlp_metadata.c b/lib/cmetrics/src/cmt_otlp_metadata.c index 81ed8171b82..f0501a58ba4 100644 --- a/lib/cmetrics/src/cmt_otlp_metadata.c +++ b/lib/cmetrics/src/cmt_otlp_metadata.c @@ -28,39 +28,110 @@ #include #include +#include +#include #define METADATA_KEY "__cmetrics_native_metadata_v1" /* Written only by unreleased builds; ignored on input, never emitted. */ #define HISTOGRAM_MARKER_KEY "__cmetrics_histogram_buckets" -struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source) -{ - mpack_writer_t writer; - mpack_reader_t reader; - struct cfl_kvlist *copy = NULL; - char *buffer = NULL; - size_t size = 0; - int result; +static struct cfl_kvlist *clone_kvlist(struct cfl_kvlist *source); - if (source == NULL) { +static struct cfl_variant *clone_variant(struct cfl_variant *source) +{ + size_t index; + struct cfl_array *array; + struct cfl_kvlist *kvlist; + struct cfl_variant *entry; + struct cfl_variant *copy; + + switch (source->type) { + case CFL_VARIANT_STRING: + return cfl_variant_create_from_string_s(source->data.as_string, + cfl_variant_size_get(source), CFL_FALSE); + case CFL_VARIANT_BYTES: + return cfl_variant_create_from_bytes(source->data.as_bytes, + cfl_variant_size_get(source), CFL_FALSE); + case CFL_VARIANT_BOOL: + return cfl_variant_create_from_bool(source->data.as_bool); + case CFL_VARIANT_INT: + return cfl_variant_create_from_int64(source->data.as_int64); + case CFL_VARIANT_UINT: + return cfl_variant_create_from_uint64(source->data.as_uint64); + case CFL_VARIANT_DOUBLE: + return cfl_variant_create_from_double(source->data.as_double); + case CFL_VARIANT_NULL: + return cfl_variant_create_from_null(); + case CFL_VARIANT_ARRAY: + array = cfl_array_create(source->data.as_array->entry_count); + if (array == NULL) { + return NULL; + } + for (index = 0; index < source->data.as_array->entry_count; index++) { + entry = clone_variant(source->data.as_array->entries[index]); + if (entry == NULL || cfl_array_append(array, entry) != 0) { + cfl_variant_destroy(entry); + cfl_array_destroy(array); + return NULL; + } + } + copy = cfl_variant_create_from_array(array); + if (copy == NULL) { + cfl_array_destroy(array); + } + return copy; + case CFL_VARIANT_KVLIST: + kvlist = clone_kvlist(source->data.as_kvlist); + if (kvlist == NULL) { + return NULL; + } + copy = cfl_variant_create_from_kvlist(kvlist); + if (copy == NULL) { + cfl_kvlist_destroy(kvlist); + } + return copy; + default: return NULL; } - mpack_writer_init_growable(&writer, &buffer, &size); - result = pack_cfl_variant_kvlist(&writer, source); - if (mpack_writer_destroy(&writer) != mpack_ok || result != 0) { - free(buffer); +} + +static struct cfl_kvlist *clone_kvlist(struct cfl_kvlist *source) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_kvlist *copy; + struct cfl_variant *value; + + copy = cfl_kvlist_create(); + if (copy == NULL) { return NULL; } - mpack_reader_init_data(&reader, buffer, size); - result = unpack_cfl_kvlist(&reader, ©); - if (mpack_reader_destroy(&reader) != mpack_ok || result != 0) { - cfl_kvlist_destroy(copy); - copy = NULL; + cfl_list_foreach(head, &source->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + value = clone_variant(pair->val); + if (value == NULL || + cfl_kvlist_insert_s(copy, pair->key, cfl_sds_len(pair->key), value) != 0) { + cfl_variant_destroy(value); + cfl_kvlist_destroy(copy); + return NULL; + } } - free(buffer); return copy; } +/* + * Deep copy, preserving key order and duplicate keys. Unlike a msgpack round + * trip it has no nesting limit, so metadata accepted by a decoder can always + * be copied. + */ +struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source) +{ + if (source == NULL) { + return NULL; + } + return clone_kvlist(source); +} + static int visit_maps(struct cmt *context, int (*callback)(struct cmt_map *, void *), void *data) { @@ -261,6 +332,201 @@ int cmt_otlp_metadata_pack(mpack_writer_t *writer, struct cmt *context) return 0; } +/* + * Fluent Bit 5.1 and older keep OTLP point metadata in the external metadata + * tree: otlp -> metrics -> -> -> {metadata, datapoints}, where a + * point is keyed by ":". They do not read the binary + * sidecar, so the encoder also writes that tree for families with metadata, + * with the keys those readers use. The tree is built on a copy: the context + * itself is not modified. + */ +static const char *legacy_point_keys[] = { + "start_time_unix_nano", "flags", "number_value_case", "has_sum", "has_min", + "min", "has_max", "max", "exemplars", NULL +}; + +static const char *legacy_type_key(int map_type) +{ + switch (map_type) { + case CMT_COUNTER: + return "counter"; + case CMT_GAUGE: + return "gauge"; + case CMT_UNTYPED: + return "untyped"; + case CMT_SUMMARY: + return "summary"; + case CMT_HISTOGRAM: + return "histogram"; + case CMT_EXP_HISTOGRAM: + return "exp_histogram"; + default: + return "unknown"; + } +} + +static struct cfl_kvlist *legacy_child(struct cfl_kvlist *parent, const char *key) +{ + struct cfl_variant *value; + struct cfl_kvlist *child; + + value = cfl_kvlist_fetch(parent, (char *) key); + if (value != NULL && value->type == CFL_VARIANT_KVLIST) { + return value->data.as_kvlist; + } + if (value != NULL) { + cfl_kvlist_remove(parent, (char *) key); + } + child = cfl_kvlist_create(); + if (child == NULL) { + return NULL; + } + if (cfl_kvlist_insert_kvlist(parent, (char *) key, child) != 0) { + cfl_kvlist_destroy(child); + return NULL; + } + return child; +} + +static int legacy_add_point(struct cfl_kvlist *datapoints, struct cmt_metric *sample) +{ + char key[64]; + size_t index; + struct cfl_variant *value; + struct cfl_variant *copy; + struct cfl_kvlist *point; + + if (sample->otlp_metadata == NULL) { + return 0; + } + + snprintf(key, sizeof(key), "%" PRIx64 ":%" PRIu64, + sample->hash, cmt_metric_get_timestamp(sample)); + cfl_kvlist_remove(datapoints, key); + point = legacy_child(datapoints, key); + if (point == NULL) { + return -1; + } + + for (index = 0; legacy_point_keys[index] != NULL; index++) { + value = cfl_kvlist_fetch(sample->otlp_metadata, (char *) legacy_point_keys[index]); + if (value == NULL) { + continue; + } + copy = clone_variant(value); + if (copy == NULL || + cfl_kvlist_insert(point, (char *) legacy_point_keys[index], copy) != 0) { + cfl_variant_destroy(copy); + return -1; + } + } + return 0; +} + +static int legacy_add_map(struct cmt_map *map, void *data) +{ + struct cfl_kvlist *metrics = data; + struct cfl_kvlist *type_root; + struct cfl_kvlist *family; + struct cfl_kvlist *datapoints; + struct cfl_kvlist *copy; + struct cfl_variant *metadata; + struct cfl_list *head; + struct cmt_metric *sample; + int points; + + metadata = map->otlp_metadata ? cfl_kvlist_fetch(map->otlp_metadata, "metadata") : NULL; + points = map->metric_static_set && map->metric.otlp_metadata != NULL; + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + points |= sample->otlp_metadata != NULL; + } + if (!points && (metadata == NULL || metadata->type != CFL_VARIANT_KVLIST)) { + return 0; + } + + type_root = legacy_child(metrics, legacy_type_key(map->type)); + if (type_root == NULL) { + return -1; + } + family = legacy_child(type_root, map->opts->fqname); + if (family == NULL) { + return -1; + } + + if (metadata != NULL && metadata->type == CFL_VARIANT_KVLIST) { + copy = clone_kvlist(metadata->data.as_kvlist); + cfl_kvlist_remove(family, "metadata"); + if (copy == NULL || cfl_kvlist_insert_kvlist(family, "metadata", copy) != 0) { + cfl_kvlist_destroy(copy); + return -1; + } + } + + if (!points) { + return 0; + } + datapoints = legacy_child(family, "datapoints"); + if (datapoints == NULL) { + return -1; + } + if (map->metric_static_set && legacy_add_point(datapoints, &map->metric) != 0) { + return -1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (legacy_add_point(datapoints, sample) != 0) { + return -1; + } + } + return 0; +} + +static int has_legacy_metadata(struct cmt_map *map, void *data) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + if ((map->otlp_metadata && cfl_kvlist_fetch(map->otlp_metadata, "metadata")) || + (map->metric_static_set && map->metric.otlp_metadata)) { + return 1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (sample->otlp_metadata) { + return 1; + } + } + return 0; +} + +int cmt_otlp_metadata_pack_external(mpack_writer_t *writer, struct cmt *context) +{ + int result; + struct cfl_kvlist *external; + struct cfl_kvlist *otlp; + struct cfl_kvlist *metrics; + + if (visit_maps(context, has_legacy_metadata, NULL) == 0) { + return pack_cfl_variant_kvlist(writer, context->external_metadata); + } + + external = clone_kvlist(context->external_metadata); + if (external == NULL) { + return -1; + } + otlp = legacy_child(external, "otlp"); + metrics = otlp ? legacy_child(otlp, "metrics") : NULL; + if (metrics == NULL || visit_maps(context, legacy_add_map, metrics) != 0) { + cfl_kvlist_destroy(external); + return -1; + } + + result = pack_cfl_variant_kvlist(writer, external); + cfl_kvlist_destroy(external); + return result; +} + struct staged_family_metadata { struct cmt_map *map; struct cfl_kvlist *family; @@ -320,12 +586,17 @@ static int stage_map_metadata(struct cmt_map *map, void *data) staged->map = map; points = cfl_kvlist_fetch(family->data.as_kvlist, "points"); reset = cfl_kvlist_fetch(family->data.as_kvlist, "allow_reset"); - if (!reset || reset->type != CFL_VARIANT_BOOL || - restore_optional_map(&staged->family, cfl_kvlist_fetch(family->data.as_kvlist, "family"))) { + if (!reset || reset->type != CFL_VARIANT_BOOL) { return -1; } staged->allow_reset = reset->data.as_bool; + /* metadata that cannot be read back is skipped for this family only */ + if (restore_optional_map(&staged->family, + cfl_kvlist_fetch(family->data.as_kvlist, "family")) != 0) { + staged->family = NULL; + } + /* Label-less entries fold into the static metric on decode, so the point * list of that family may no longer line up: keep the family metadata * but skip its point metadata rather than misattributing it. */ @@ -343,8 +614,10 @@ static int stage_map_metadata(struct cmt_map *map, void *data) } } for (index = 0; index < staged->point_count; index++) { - if (restore_optional_map(&staged->points[index], points->data.as_array->entries[index])) { - return -1; + /* a point whose metadata cannot be read back keeps none */ + if (restore_optional_map(&staged->points[index], + points->data.as_array->entries[index]) != 0) { + staged->points[index] = NULL; } } return 0; @@ -413,6 +686,27 @@ static int check_map_metadata_shape(struct cmt_map *map, void *data) return 0; } +static void drop_legacy_family(struct cmt *context, struct cmt_map *map) +{ + struct cfl_variant *otlp; + struct cfl_variant *metrics; + struct cfl_variant *type_root; + + otlp = cfl_kvlist_fetch(context->external_metadata, "otlp"); + if (otlp == NULL || otlp->type != CFL_VARIANT_KVLIST) { + return; + } + metrics = cfl_kvlist_fetch(otlp->data.as_kvlist, "metrics"); + if (metrics == NULL || metrics->type != CFL_VARIANT_KVLIST) { + return; + } + type_root = cfl_kvlist_fetch(metrics->data.as_kvlist, (char *) legacy_type_key(map->type)); + if (type_root == NULL || type_root->type != CFL_VARIANT_KVLIST) { + return; + } + cfl_kvlist_remove(type_root->data.as_kvlist, map->opts->fqname); +} + int cmt_otlp_metadata_restore(struct cmt *context) { struct cfl_variant *value; @@ -446,6 +740,8 @@ int cmt_otlp_metadata_restore(struct cmt *context) result = visit_maps(context, stage_map_metadata, &state); if (result == 0) { for (family_index = 0; family_index < state.families->entry_count; family_index++) { + /* the legacy copy written for older readers is superseded */ + drop_legacy_family(context, state.staged[family_index].map); commit_map_metadata(&state.staged[family_index]); } } diff --git a/lib/cmetrics/tests/opentelemetry.c b/lib/cmetrics/tests/opentelemetry.c index 99dfc35fada..0bf8221ec37 100644 --- a/lib/cmetrics/tests/opentelemetry.c +++ b/lib/cmetrics/tests/opentelemetry.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -25,7 +26,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -2851,6 +2854,169 @@ static void test_otlp_unsigned_attribute_preserves_signed_range(void) cmt_destroy(ctx); } +/* Metadata deeper than the msgpack nesting limit stays usable. */ +static void test_otlp_metadata_deep_nesting(void) +{ + int depth; + int found = 0; + struct cfl_list *head; + struct cmt_gauge *entry; + char *buffer; + size_t size; + size_t offset; + struct cmt *ctx; + struct cmt *copy; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *shallow; + struct cmt_metric *sample; + struct cfl_kvlist *inner; + struct cfl_kvlist *outer; + char *labels[] = {"a"}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "deep", "deep", 1, (char *[]) {"k"}); + shallow = cmt_gauge_create(ctx, "", "", "shallow", "shallow", 0, NULL); + TEST_ASSERT(gauge != NULL && shallow != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 1000, 7, 1, labels) == 0); + TEST_ASSERT(cmt_gauge_set(shallow, 1000, 9, 0, NULL) == 0); + + /* a point whose metadata nests 40 maps deep */ + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + inner = cfl_kvlist_create(); + TEST_ASSERT(inner != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(inner, "leaf", "value") == 0); + for (depth = 0; depth < 40; depth++) { + outer = cfl_kvlist_create(); + TEST_ASSERT(outer != NULL); + TEST_ASSERT(cfl_kvlist_insert_kvlist(outer, "nested", inner) == 0); + inner = outer; + } + sample->otlp_metadata = inner; + + shallow->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(shallow->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(shallow->map->metric.otlp_metadata, + "start_time_unix_nano", 500) == 0); + + /* copying a context copies the deep metadata too */ + copy = cmt_create(); + TEST_ASSERT(copy != NULL); + TEST_CHECK(cmt_cat(copy, ctx) == 0); + gauge = cfl_list_entry_first(©->gauges, struct cmt_gauge, _head); + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_CHECK(sample->otlp_metadata != NULL && + cfl_kvlist_fetch(sample->otlp_metadata, "nested") != NULL); + cmt_destroy(copy); + + /* a msgpack round trip skips only the point it cannot read back */ + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + offset = 0; + decoded = NULL; + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + cfl_list_foreach(head, &decoded->gauges) { + entry = cfl_list_entry(head, struct cmt_gauge, _head); + if (strcmp(entry->opts.name, "shallow") == 0) { + found++; + TEST_CHECK(entry->map->metric.otlp_metadata != NULL && + cfl_kvlist_fetch(entry->map->metric.otlp_metadata, + "start_time_unix_nano") != NULL); + } + else if (strcmp(entry->opts.name, "deep") == 0) { + found++; + sample = cfl_list_entry_first(&entry->map->metrics, struct cmt_metric, _head); + TEST_CHECK(sample->otlp_metadata == NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 7); + } + } + TEST_CHECK(found == 2); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Older readers find point metadata in the external tree, keyed by series. */ +static void test_otlp_metadata_legacy_external_tree(void) +{ + char key[64]; + char *buffer = NULL; + size_t size = 0; + mpack_writer_t writer; + mpack_reader_t reader; + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *family; + struct cfl_kvlist *metadata; + struct cfl_kvlist *external = NULL; + struct cfl_variant *node; + struct cfl_variant *point; + char *labels[] = {"a"}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "legacy", "legacy", 1, (char *[]) {"k"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 1000, 7, 1, labels) == 0); + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_ASSERT(sample != NULL); + + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "start_time_unix_nano", 500) == 0); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", 1) == 0); + TEST_ASSERT(cfl_kvlist_insert_string(sample->otlp_metadata, "number_value_case", "double") == 0); + TEST_ASSERT(cmt_otlp_capture_labels(gauge->map, sample) == 0); + + gauge->map->otlp_metadata = cfl_kvlist_create(); + metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(metadata, "unit.source", "otlp") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", metadata) == 0); + + mpack_writer_init_growable(&writer, &buffer, &size); + TEST_ASSERT(cmt_otlp_metadata_pack_external(&writer, ctx) == 0); + TEST_ASSERT(mpack_writer_destroy(&writer) == mpack_ok); + mpack_reader_init_data(&reader, buffer, size); + TEST_ASSERT(unpack_cfl_kvlist(&reader, &external) == 0); + mpack_reader_destroy(&reader); + free(buffer); + + node = cfl_kvlist_fetch(external, "otlp"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(node->data.as_kvlist, "metrics"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(node->data.as_kvlist, "gauge"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(node->data.as_kvlist, "legacy"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + family = node->data.as_kvlist; + + node = cfl_kvlist_fetch(family, "metadata"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + TEST_CHECK(cfl_kvlist_fetch(node->data.as_kvlist, "unit.source") != NULL); + + node = cfl_kvlist_fetch(family, "datapoints"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + snprintf(key, sizeof(key), "%" PRIx64 ":%" PRIu64, + sample->hash, cmt_metric_get_timestamp(sample)); + point = cfl_kvlist_fetch(node->data.as_kvlist, key); + TEST_ASSERT(point != NULL && point->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(point->data.as_kvlist, "start_time_unix_nano"); + TEST_CHECK(node != NULL && node->data.as_uint64 == 500); + node = cfl_kvlist_fetch(point->data.as_kvlist, "flags"); + TEST_CHECK(node != NULL && node->data.as_uint64 == 1); + /* only the keys older readers use are written */ + TEST_CHECK(cfl_kvlist_fetch(point->data.as_kvlist, "attributes") == NULL); + + /* the context itself is unchanged */ + TEST_CHECK(cfl_kvlist_fetch(ctx->external_metadata, "otlp") == NULL); + + cfl_kvlist_destroy(external); + cmt_destroy(ctx); +} + /* Typed attributes without an OTLP form keep the label derived attributes. */ static void test_otlp_unrepresentable_attribute_keeps_point(void) { @@ -3056,6 +3222,8 @@ static void test_opentelemetry_repeated_unlabeled_points_last_wins(void) TEST_LIST = { {"otlp_unrepresentable_attribute_keeps_point", test_otlp_unrepresentable_attribute_keeps_point}, + {"otlp_metadata_legacy_external_tree", test_otlp_metadata_legacy_external_tree}, + {"otlp_metadata_deep_nesting", test_otlp_metadata_deep_nesting}, {"repeated_unlabeled_points_last_wins", test_opentelemetry_repeated_unlabeled_points_last_wins}, {"otlp_unsigned_attribute_signed_range", test_otlp_unsigned_attribute_preserves_signed_range}, {"opentelemetry_api_full_roundtrip_with_msgpack", test_opentelemetry_api_full_roundtrip_with_msgpack}, From 550e9bcbf910ba82e70e4be3655afe93b8953acb Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:00 -0600 Subject: [PATCH 51/61] lib: ctraces: store unknown span kinds as unspecified Releases up to 5.1 reject a whole traces context when a span kind is out of the known range. The OTLP decoders still keep the spans with unknown kinds, and msgpack stores those kinds as unspecified. Signed-off-by: Eduardo Silva --- lib/ctraces/src/ctr_encode_msgpack.c | 13 +++++++++++-- lib/ctraces/tests/decoding.c | 13 +++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/ctraces/src/ctr_encode_msgpack.c b/lib/ctraces/src/ctr_encode_msgpack.c index 43f11b033ba..b2bf014cfbe 100644 --- a/lib/ctraces/src/ctr_encode_msgpack.c +++ b/lib/ctraces/src/ctr_encode_msgpack.c @@ -361,9 +361,18 @@ static void pack_span(mpack_writer_t *writer, struct ctrace_span *span) mpack_write_nil(writer); } - /* kind */ + /* + * kind: decoders before unknown kinds were preserved reject a whole + * context when the value is outside the known range, so unknown kinds + * are stored as unspecified. + */ mpack_write_cstr(writer, "kind"); - mpack_write_i32(writer, span->kind); + if (span->kind >= CTRACE_SPAN_UNSPECIFIED && span->kind <= CTRACE_SPAN_CONSUMER) { + mpack_write_u32(writer, span->kind); + } + else { + mpack_write_u32(writer, CTRACE_SPAN_UNSPECIFIED); + } /* start_time_unix_nano */ mpack_write_cstr(writer, "start_time_unix_nano"); diff --git a/lib/ctraces/tests/decoding.c b/lib/ctraces/tests/decoding.c index bee8046600e..5e7cf9cc986 100644 --- a/lib/ctraces/tests/decoding.c +++ b/lib/ctraces/tests/decoding.c @@ -625,7 +625,7 @@ void test_msgpack_preserves_flags() ctr_destroy(ctx); } -void test_msgpack_preserves_signed_kind() +void test_msgpack_stores_unknown_kind_as_unspecified() { char *buffer; size_t size; @@ -647,7 +647,7 @@ void test_msgpack_preserves_signed_kind() TEST_ASSERT(ss != NULL); span = ctr_span_create(ctx, ss, "unknown-kind", NULL); TEST_ASSERT(span != NULL); - /* Decoders preserve unknown protobuf enum values. */ + /* OTLP decoders keep unknown enum values, msgpack stores them as unspecified */ span->kind = kinds[index]; TEST_ASSERT(ctr_span_create(ctx, ss, "healthy-sibling", NULL) != NULL); TEST_ASSERT(ctr_encode_msgpack_create(ctx, &buffer, &size) == 0); @@ -659,7 +659,12 @@ void test_msgpack_preserves_signed_kind() struct ctrace_resource_span, _head); ss = cfl_list_entry(rs->scope_spans.next, struct ctrace_scope_span, _head); span = cfl_list_entry(ss->spans.next, struct ctrace_span, _head); - TEST_CHECK(span->kind == kinds[index]); + if (kinds[index] >= CTRACE_SPAN_UNSPECIFIED && kinds[index] <= CTRACE_SPAN_CONSUMER) { + TEST_CHECK(span->kind == kinds[index]); + } + else { + TEST_CHECK(span->kind == CTRACE_SPAN_UNSPECIFIED); + } TEST_CHECK(cfl_list_size(&ss->spans) == 2); TEST_CHECK(offset == size); ctr_destroy(decoded); @@ -1057,7 +1062,7 @@ void test_simple_to_msgpack_and_back() TEST_LIST = { - {"msgpack_preserves_signed_kind", test_msgpack_preserves_signed_kind}, + {"msgpack_unknown_kind_unspecified", test_msgpack_stores_unknown_kind_as_unspecified}, {"cmt_simple_to_msgpack_and_back", test_simple_to_msgpack_and_back}, {"cmt_msgpack", test_msgpack_to_cmt}, {"empty_spans", test_msgpack_to_ctr_with_empty_spans}, From 9f37c946a75b4a8b60cc6267ba07d0196c05d115 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:00 -0600 Subject: [PATCH 52/61] opentelemetry: free metric arrays when summary allocation fails Signed-off-by: Eduardo Silva --- src/opentelemetry/flb_opentelemetry_metrics.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/opentelemetry/flb_opentelemetry_metrics.c b/src/opentelemetry/flb_opentelemetry_metrics.c index 43695aa455f..039b53a17c5 100644 --- a/src/opentelemetry/flb_opentelemetry_metrics.c +++ b/src/opentelemetry/flb_opentelemetry_metrics.c @@ -2770,6 +2770,8 @@ static int process_metric_summary_data_points(struct cmt *context, if (sample->sum_quantiles == NULL) { destroy_label_arrays(point_label_count, NULL, point_label_values); flb_free(point_quantile_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_quantiles); return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; } sample->sum_quantiles_count = summary->quantiles_count; From e2587deb649843b681ccdab7352dec93c290de02 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:00 -0600 Subject: [PATCH 53/61] tests: internal: processor: cover group boundaries of the input Signed-off-by: Eduardo Silva --- tests/internal/processor.c | 133 +++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) diff --git a/tests/internal/processor.c b/tests/internal/processor.c index 0880d896af7..f400a57e7af 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -1309,6 +1309,138 @@ static void processor_chunk_codec_is_per_run(void) window_config_destroy(config); } +/* removes group end markers, as a filtering processor can do */ +static int drop_group_ends(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + int type; + struct cfl_list *head; + struct cfl_list *tmp; + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_record *record; + + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + } + chunk->record_pos = NULL; + + cfl_list_foreach_safe(head, tmp, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + if (flb_log_event_decoder_get_record_type(&record->event, &type) == 0 && + type == FLB_LOG_EVENT_GROUP_END) { + flb_mp_chunk_cobj_record_destroy(chunk, record); + } + } + + return FLB_PROCESSOR_SUCCESS; +} + +static struct flb_processor_plugin drop_group_end_plugin = { + .name = "test_drop_group_end", + .description = "removes group end markers", + .cb_process_logs = drop_group_ends, + .flags = 0 +}; + +/* layout tokens: r record, S group start, E group end */ +static void run_group_layout(struct flb_config *config, char *unit, + const char *layout, const char *expected) +{ + int ret; + int type; + int length; + char token[16]; + char result[128]; + const char *cursor; + void *output = NULL; + size_t output_size = 0; + msgpack_object *seq; + struct flb_processor *proc; + struct flb_input_instance owner; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + struct flb_log_event_encoder *encoder; + + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + proc = window_processor_create(config, &owner, unit, NULL); + TEST_ASSERT(proc != NULL); + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (cursor = layout; sscanf(cursor, "%15s%n", token, &length) == 1; cursor += length) { + if (token[0] == 'r') { + TEST_ASSERT(append_window_record(encoder, atoi(token + 1)) == 0); + } + else if (token[0] == 'S') { + TEST_ASSERT(flb_log_event_encoder_group_init(encoder) == 0); + TEST_ASSERT(flb_log_event_encoder_group_header_end(encoder) == 0); + } + else { + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + } + } + + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, + &output, &output_size); + TEST_CHECK(ret == 0); + + result[0] = '\0'; + if (output != NULL && output_size > 0) { + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type == FLB_LOG_EVENT_GROUP_START) { + snprintf(token, sizeof(token), "S "); + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + snprintf(token, sizeof(token), "E "); + } + else { + seq = &event.body->via.map.ptr[0].val; + snprintf(token, sizeof(token), "r%d ", (int) seq->via.i64); + } + strncat(result, token, sizeof(result) - strlen(result) - 1); + } + flb_log_event_decoder_destroy(&decoder); + } + if (strlen(result) > 0) { + result[strlen(result) - 1] = '\0'; + } + TEST_CHECK(strcmp(result, expected) == 0); + TEST_MSG("%s: '%s' produced '%s', expected '%s'", unit, layout, result, expected); + + if (output != NULL && output != encoder->output_buffer) { + flb_free(output); + } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(proc); + cmt_destroy(owner.cmt); +} + +static void processor_group_boundaries_follow_the_input(void) +{ + struct flb_config *config; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + mk_list_add(&drop_group_end_plugin._head, &config->processor_plugins); + + /* a processor removing an end never leaves an orphan start behind */ + run_group_layout(config, "test_drop_group_end", "S r0 r1 E r2", "r0 r1 r2"); + /* groups split across appends keep their boundary markers */ + run_group_layout(config, "test_single_mark", "S r0 r1", "S r0 r1"); + run_group_layout(config, "test_single_mark", "r0 E S r1 E", "r0 E S r1 E"); + /* stray and empty markers of a balanced input are removed as before */ + run_group_layout(config, "test_single_mark", "S r0 E E r1", "S r0 E r1"); + + mk_list_del(&drop_group_end_plugin._head); + window_config_destroy(config); +} + TEST_LIST = { {"processor_metrics_failure_clears_output", processor_metrics_failure_clears_output}, { "processor_private_inputs_use_main_loop", processor_private_inputs_use_main_loop }, @@ -1319,5 +1451,6 @@ TEST_LIST = { { "processor_log_windows_drop_all", processor_log_windows_drop_all }, { "processor_log_windows_need_every_unit", processor_log_windows_need_every_unit }, { "processor_chunk_codec_is_per_run", processor_chunk_codec_is_per_run }, + { "processor_group_boundaries_follow_the_input", processor_group_boundaries_follow_the_input }, { 0 } }; From cda1fa3fe72345189069c6663428c03326cbcbab Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:00 -0600 Subject: [PATCH 54/61] tests: integration: expect unknown span kinds as unspecified when stored Signed-off-by: Eduardo Silva --- .../scenarios/processor_fts/tests/test_fts_native_interop.py | 3 ++- .../scenarios/processor_fts/tests/test_fts_security_batches.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py b/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py index c773175cfdc..95c4553945c 100644 --- a/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py @@ -245,7 +245,8 @@ def test_without_fts_unknown_signed_span_kind_keeps_siblings(tmp_path, export): Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) rows = [span for batch in batches for resource in batch.get("resourceSpans", []) for scope in resource["scopeSpans"] for span in scope["spans"]] - assert [(row["name"], row["kind"]) for row in rows] == [("negative-kind", -1), ("healthy", 2)] + # stored traces keep the span; an unknown kind is stored as unspecified + assert [(row["name"], row.get("kind", 0)) for row in rows] == [("negative-kind", 0), ("healthy", 2)] diagnostics = Path(service.flb.log_file).read_text() assert "could not decode" not in diagnostics and "could not convert" not in diagnostics finally: diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py b/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py index 9b6051f5f77..0be7dcf8975 100644 --- a/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py @@ -185,7 +185,8 @@ def test_unknown_inherited_kind_does_not_bypass_privacy(batch_runner): items[0]["parentSpanId"] = "0" * 16 items[0]["endTimeUnixNano"] = "1" _, rows = batch_runner.send(items) - assert len(rows) == 2 and rows[0]["kind"] == 9 + # the span is kept; stored traces record an unknown kind as unspecified + assert len(rows) == 2 and rows[0].get("kind", 0) == 0 assert all(attributes(r["attributes"])["secret"] == "[REDACTED]" for r in rows) assert "edits rolled back" not in batch_runner.recent_log() From e0d9a0e17f31223b689920d881caeaaba664da63 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 16 Oct 2026 23:04:04 -0600 Subject: [PATCH 55/61] tests: internal: fts: cover kept metric family name collisions Signed-off-by: Eduardo Silva --- tests/internal/fts.c | 71 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/tests/internal/fts.c b/tests/internal/fts.c index 571f999529c..8e9d917054f 100644 --- a/tests/internal/fts.c +++ b/tests/internal/fts.c @@ -654,6 +654,76 @@ static void native_metric_scalar_preservation(void) } } +static void kept_native_family_does_not_merge_into_outputs(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct cmt *input; + struct cmt *output; + struct cmt_gauge *native; + struct cmt_gauge *healthy; + struct cmt_gauge *gauge; + struct cfl_list *head; + void *result; + size_t result_size; + int seen_native = 0; + int seen_healthy = 0; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_metric_collision", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_METRICS, "fts"); + TEST_ASSERT(unit != NULL); + /* the policy output reuses the name of a family kept outside the policy */ + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", + "function process(metric) metric.name = 'native' end") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "error_mode", "keep") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + + input = cmt_create(); + TEST_ASSERT(input != NULL); + native = cmt_gauge_create(input, "", "", "native", "shared", 0, NULL); + TEST_ASSERT(native != NULL); + native->map->metric_static_set = CMT_TRUE; + cmt_metric_set_uint64(&native->map->metric, 100, UINT64_MAX); + healthy = cmt_gauge_create(input, "", "", "healthy", "shared", 0, NULL); + TEST_ASSERT(healthy != NULL); + TEST_ASSERT(cmt_gauge_set(healthy, 100, 42, 0, NULL) == 0); + + result = NULL; + result_size = 0; + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_METRICS, "native", 6, + input, 0, &result, &result_size) == 0); + output = result; + TEST_ASSERT(output != NULL); + + /* the batch falls back to its original families instead of merging them */ + TEST_CHECK(cfl_list_size(&output->gauges) == 2); + cfl_list_foreach(head, &output->gauges) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + if (strcmp(gauge->opts.name, "native") == 0) { + seen_native++; + TEST_CHECK(cmt_metric_get_value_type(&gauge->map->metric) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK(cmt_metric_get_uint64_value(&gauge->map->metric) == UINT64_MAX); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 0); + } + else if (strcmp(gauge->opts.name, "healthy") == 0) { + seen_healthy++; + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 42); + } + } + TEST_CHECK(seen_native == 1 && seen_healthy == 1); + + if (output != input) { + cmt_destroy(output); + } + cmt_destroy(input); + flb_processor_destroy(processor); + flb_config_exit(config); +} + static void untouched_otel_fields_are_preserved(void) { run_policy("function process(log) end", 3, "drop", 6, 2, 0, NULL, NULL); @@ -1415,6 +1485,7 @@ TEST_LIST = { {"trusted_policy_jit_compiles", trusted_policy_jit_compiles}, {"native_histogram_buckets_are_preserved", native_histogram_buckets_are_preserved}, {"native_metric_scalar_preservation", native_metric_scalar_preservation}, + {"kept_native_family_no_merge", kept_native_family_does_not_merge_into_outputs}, {"generic_context_isolation", generic_context_isolation}, {"generic_read_does_not_split", generic_read_does_not_split}, {"plain_logs_do_not_gain_context", plain_logs_do_not_gain_context}, From 632a9f87edb87258d03cf96d0f94a0dfc6de82b9 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 17 Oct 2026 06:35:24 -0600 Subject: [PATCH 56/61] tests: internal: processor: get pthreads from flb_compat.h The per-run codec test included pthread.h directly, which MSVC does not provide. flb_compat.h maps it to the bundled winpthreads on Windows. Signed-off-by: Eduardo Silva --- tests/internal/processor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/internal/processor.c b/tests/internal/processor.c index f400a57e7af..d77fba9d86f 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -18,7 +18,7 @@ */ #include -#include +#include #include #include #include From 73c9f65079a6b47493105879192c3509ada71faa Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 17 Oct 2026 07:17:07 -0600 Subject: [PATCH 57/61] out_opentelemetry: send every pending log record once The final export of a chunk only checked the record count of the last scope, which restarts at zero for every scope. A chunk that ended with a group start and no records yet, such as a group whose records arrive in a later append, dropped all its earlier records while the flush reported success. Count the records pending since the last export instead. A batch export also released only the records of the current scope, so records of earlier scopes in the same request were sent again with the next batch. Release the records of every scope after an export. Signed-off-by: Eduardo Silva --- .../out_opentelemetry/opentelemetry_logs.c | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/plugins/out_opentelemetry/opentelemetry_logs.c b/plugins/out_opentelemetry/opentelemetry_logs.c index f3a9835ece6..f790a4fde7b 100644 --- a/plugins/out_opentelemetry/opentelemetry_logs.c +++ b/plugins/out_opentelemetry/opentelemetry_logs.c @@ -912,6 +912,36 @@ static void free_log_records(Opentelemetry__Proto__Logs__V1__LogRecord **logs, s } } +/* + * Release the log records of a request that was already sent, keeping its + * resources and scopes, so a later request of the same chunk does not repeat + * them. + */ +static void release_sent_log_records(Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *export_logs) +{ + size_t i; + size_t j; + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; + + for (i = 0; i < export_logs->n_resource_logs; i++) { + resource_log = export_logs->resource_logs[i]; + if (resource_log == NULL) { + continue; + } + + for (j = 0; j < resource_log->n_scope_logs; j++) { + scope_log = resource_log->scope_logs[j]; + if (scope_log == NULL || scope_log->log_records == NULL) { + continue; + } + + free_log_records(scope_log->log_records, scope_log->n_log_records); + scope_log->n_log_records = 0; + } + } +} + static void free_resource_logs(Opentelemetry__Proto__Logs__V1__ResourceLogs **resource_logs, size_t resource_count) { int i; @@ -1182,6 +1212,7 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, int ret; int record_type; int log_record_count; + int pending_records = 0; int max_scopes_limit; int max_resources; int native_otel = FLB_FALSE; @@ -1636,13 +1667,14 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, ret = FLB_OK; log_record_count++; + pending_records++; scope_log->n_log_records = log_record_count; if (log_record_count >= ctx->batch_size) { ret = logs_flush_to_otel(ctx, event_chunk, &export_logs); - free_log_records(log_records, log_record_count); + release_sent_log_records(&export_logs); log_record_count = 0; - scope_log->n_log_records = 0; + pending_records = 0; /* Do not let a later batch overwrite this failure. */ if (ret != FLB_OK) { @@ -1653,7 +1685,11 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, flb_log_event_decoder_destroy(decoder); - if (log_record_count > 0 && ret == FLB_OK) { + /* + * Records of earlier scopes are pending too: a chunk can end with a group + * that has no records yet, for example one opened by the previous append. + */ + if (pending_records > 0 && ret == FLB_OK) { ret = logs_flush_to_otel(ctx, event_chunk, &export_logs); } From 32439bb959efb9c949048090914b820f082c518c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 17 Oct 2026 07:17:07 -0600 Subject: [PATCH 58/61] tests: integration: out_opentelemetry: cover empty trailing groups and batches Signed-off-by: Eduardo Silva --- ...out_otel_http_logs_forward_batch_size.yaml | 18 ++++ .../out_otel_http_logs_forward_groups.yaml | 17 ++++ .../tests/test_out_opentelemetry_001.py | 89 +++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_batch_size.yaml create mode 100644 tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_groups.yaml diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_batch_size.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_batch_size.yaml new file mode 100644 index 00000000000..0fdad8c9533 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_batch_size.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + batch_size: 3 diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_groups.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_groups.yaml new file mode 100644 index 00000000000..2d327081491 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_groups.yaml @@ -0,0 +1,17 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py b/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py index 0481f64b65c..647131c6720 100644 --- a/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py +++ b/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py @@ -6,6 +6,7 @@ import shutil import socket import threading +import time from collections import Counter import requests @@ -1454,3 +1455,91 @@ def test_out_opentelemetry_logs_max_scopes_enforcement(): output = json.loads(json_format.MessageToJson(logs_seen[0])) assert len(output["resourceLogs"]) == 4 assert all(len(resource_log["scopeLogs"]) == 1 for resource_log in output["resourceLogs"]) + + +def _pack(value): + """Encode forward fixtures without an optional MessagePack dependency.""" + if isinstance(value, bytes): + return value + if isinstance(value, int): + return b"\xcf" + value.to_bytes(8, "big") + if isinstance(value, str): + data = value.encode("utf-8") + return b"\xdb" + len(data).to_bytes(4, "big") + data + if isinstance(value, list): + return b"\xdd" + len(value).to_bytes(4, "big") + b"".join(_pack(item) for item in value) + if isinstance(value, dict): + return b"\xdf" + len(value).to_bytes(4, "big") + b"".join( + _pack(key) + _pack(item) for key, item in value.items()) + raise TypeError(type(value)) + + +def _event_time(seconds): + return b"\xd7\x00" + seconds.to_bytes(4, "big") + bytes(4) + + +def _group_start(resource_id, name): + return [[_event_time(0xFFFFFFFF), {"schema": "otlp", "resource_id": resource_id, "scope_id": 0}], + {"resource": {"attributes": {"service.name": name}}, "scope": {"name": "scope"}}] + + +def _group_end(): + return [[_event_time(0xFFFFFFFE), {}], {}] + + +def _log_record(record_id): + return [[_event_time(1700000000), {"otlp": {}}], {"id": record_id}] + + +def _send_forward(service, entries): + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=10) as connection: + connection.sendall(_pack(["test", entries])) + + +def _exported_record_ids(): + ids = [] + for export_request in data_storage["logs"]: + output = json.loads(json_format.MessageToJson(export_request)) + for record, _, _ in iter_log_records(output): + for item in record["body"]["kvlistValue"]["values"]: + if item["key"] == "id": + ids.append(int(item["value"]["intValue"])) + return sorted(ids) + + +def test_out_opentelemetry_chunk_ending_with_empty_group_exports_its_records(): + """A chunk that ends with a group start and no records still exports the earlier groups.""" + service = Service("out_otel_http_logs_forward_groups.yaml") + service.start() + try: + # the next group starts in this append and receives its records later + _send_forward(service, [_group_start(0, "first"), _log_record(0), _log_record(1), + _group_end(), _group_start(1, "second")]) + service.service.wait_for_condition( + lambda: _exported_record_ids() == [0, 1] or None, + timeout=15, interval=0.5, description="records of the first group") + + _send_forward(service, [_log_record(2), _group_end()]) + service.service.wait_for_condition( + lambda: _exported_record_ids() == [0, 1, 2] or None, + timeout=15, interval=0.5, description="record appended to the open group") + finally: + service.stop() + + +def test_out_opentelemetry_batch_size_exports_each_record_once(): + """Records of earlier scopes are not sent again by a later batch.""" + service = Service("out_otel_http_logs_forward_batch_size.yaml") + service.start() + try: + entries = [_group_start(0, "first")] + [_log_record(i) for i in range(2)] + [_group_end()] + entries += [_group_start(1, "second")] + [_log_record(i) for i in range(2, 7)] + [_group_end()] + _send_forward(service, entries) + service.service.wait_for_condition( + lambda: len(_exported_record_ids()) >= 7 or None, + timeout=15, interval=0.5, description="seven exported records") + # give a duplicated export time to arrive + time.sleep(2) + assert _exported_record_ids() == list(range(7)) + finally: + service.stop() From 71a55286dd17cb35e37dff838fc3d6cc2426aeaa Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 17 Oct 2026 08:02:10 -0600 Subject: [PATCH 59/61] tests: internal: fts: fix the MSVC build Parenthesize the expression passed to FLB_LOG_EVENT_INT64_VALUE(), whose cast otherwise binds to its first operand, and rename a local named 'small', which the Windows headers define as a macro. Signed-off-by: Eduardo Silva --- tests/internal/fts.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/internal/fts.c b/tests/internal/fts.c index 8e9d917054f..383b9f32153 100644 --- a/tests/internal/fts.c +++ b/tests/internal/fts.c @@ -779,7 +779,7 @@ static void small_record_batch_uses_wire_budget(void) TEST_CHECK(flb_log_event_encoder_begin_record(encoder) == 0); TEST_CHECK(flb_log_event_encoder_set_timestamp(encoder, ×tamp) == 0); ret = flb_log_event_encoder_append_body_values(encoder, - FLB_LOG_EVENT_CSTRING_VALUE("value"), FLB_LOG_EVENT_INT64_VALUE(index % 10)); + FLB_LOG_EVENT_CSTRING_VALUE("value"), FLB_LOG_EVENT_INT64_VALUE((index % 10))); TEST_CHECK(ret == 0); TEST_CHECK(flb_log_event_encoder_commit_record(encoder) == 0); } @@ -825,16 +825,16 @@ static void replaced_and_redacted_values_do_not_accumulate_charges(void) static int exhaust_or_recover_memory(lua_State *l) { static char payload[300000]; - char small[1024]; + char entry[1024]; size_t index; if (lua_touserdata(l, 1) == NULL) { lua_gc(l, LUA_GCSTOP, 0); lua_newtable(l); for (index = 0; index < 10000; index++) { - memset(small, 'x', sizeof(small)); - snprintf(small, 32, "%zu", index); - lua_pushlstring(l, small, sizeof(small)); + memset(entry, 'x', sizeof(entry)); + snprintf(entry, 32, "%zu", index); + lua_pushlstring(l, entry, sizeof(entry)); lua_rawseti(l, -2, index + 1); } return luaL_error(l, "expected memory exhaustion"); From 897e21a96c3964afc359bdf0de23d67da37c699d Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 17 Oct 2026 10:13:08 -0600 Subject: [PATCH 60/61] mp: count an empty log buffer without the mpack reader When an output processor drops every record, the flush counts the records of a NULL, empty buffer. The log scanner passed it to mpack, whose reader asserts on a NULL buffer, which aborts builds with mpack assertions enabled, such as the macOS unit test jobs. Signed-off-by: Eduardo Silva --- src/flb_mp.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/flb_mp.c b/src/flb_mp.c index 6570ab8ead2..d02e91281da 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -123,8 +123,13 @@ int flb_mp_scan_log_records(const void *data, size_t bytes, int *records) int count = 0; int ret = -1; - if (!data && bytes) { - return -1; + /* mpack asserts on a NULL buffer even when it is empty */ + if (!data) { + if (bytes) { + return -1; + } + *records = 0; + return 0; } mpack_reader_init_data(&reader, data, bytes); From 943f736bf5eb3098671825969509101dd173bac0 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Sat, 17 Oct 2026 10:13:08 -0600 Subject: [PATCH 61/61] tests: internal: mp: cover log helpers with empty buffers Signed-off-by: Eduardo Silva --- tests/internal/mp.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/internal/mp.c b/tests/internal/mp.c index 891cfc3eabb..bf166758c43 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -763,6 +763,32 @@ static int reference_log_count(const char *data, size_t size) return count; } +/* a processor that drops every record leaves an empty, NULL buffer */ +static void test_log_helpers_accept_empty_buffers(void) +{ + int count; + int records; + char *out_buf; + size_t out_size; + size_t window_end; + + count = -1; + TEST_CHECK(flb_mp_scan_log_records(NULL, 0, &count) == 0 && count == 0); + TEST_CHECK(flb_mp_scan_log_records(NULL, 1, &count) == -1); + TEST_CHECK(flb_mp_count_log_records(NULL, 0) == 0); + TEST_CHECK(flb_mp_log_group_boundaries(NULL, 0) == 0); + TEST_CHECK(flb_mp_log_group_starts(NULL, 0) == 0); + TEST_CHECK(flb_mp_log_window_next(NULL, 0, 0, 65536, 1024, + &window_end, &records) == -1); + + out_buf = (char *) 1; + out_size = 1; + TEST_CHECK(flb_mp_normalize_log_buffer_groups_msgpack_ex(NULL, 0, &out_buf, &out_size, + FLB_MP_GROUPS_KEEP_LEADING_END | + FLB_MP_GROUPS_KEEP_TRAILING_OPEN) == 0); + TEST_CHECK(out_buf == NULL && out_size == 0); +} + static void test_log_scan_matches_decoder(void) { struct flb_log_event_encoder *encoder; @@ -1121,6 +1147,7 @@ static void test_group_boundaries_follow_the_input(void) } TEST_LIST = { + {"log_helpers_accept_empty_buffers", test_log_helpers_accept_empty_buffers}, {"log_scan_matches_decoder", test_log_scan_matches_decoder}, {"log_window_next", test_log_window_next}, {"group_normalization_append_boundaries", test_group_normalization_keeps_append_boundaries},