diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index c0829746dc6..8c26cfe9b64 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -78,6 +78,7 @@ DEFINE_OPTION(FLB_IN_EBPF "Enable Linux eBPF input plugin" # ========== DEFINE_OPTION(FLB_PROCESSOR_CONTENT_MODIFIER "Enable content modifier processor" ON) DEFINE_OPTION(FLB_PROCESSOR_CUMULATIVE_TO_DELTA "Enable cumulative to delta metrics processor" ON) +DEFINE_OPTION(FLB_PROCESSOR_FTS "Enable Fluent Typed Scripting processor" ON) DEFINE_OPTION(FLB_PROCESSOR_LABELS "Enable metrics label manipulation processor" ON) DEFINE_OPTION(FLB_PROCESSOR_METRICS_SELECTOR "Enable metrics selector processor" ON) DEFINE_OPTION(FLB_PROCESSOR_OPENTELEMETRY_ENVELOPE "Enable OpenTelemetry envelope processor" ON) diff --git a/include/fluent-bit/flb_mp.h b/include/fluent-bit/flb_mp.h index 20a5ddcc6d4..d979362f736 100644 --- a/include/fluent-bit/flb_mp.h +++ b/include/fluent-bit/flb_mp.h @@ -27,9 +27,28 @@ #define FLB_MP_ARRAY MSGPACK_OBJECT_ARRAY int flb_mp_count(const void *data, size_t bytes); + +/* + * Count log records without building msgpack objects; allocation-free in + * non-tracking builds. Rejects unsupported or deep shapes, so callers can fall + * back to the event decoder. + */ +int flb_mp_scan_log_records(const void *data, size_t bytes, int *records); int flb_mp_count_log_records(const void *data, size_t bytes); +int flb_mp_log_window_next(const void *data, size_t bytes, size_t offset, + size_t limit_bytes, int limit_records, + size_t *window_end, int *window_records); +/* group markers that belong to neighbouring appends, see flb_mp_log_group_boundaries() */ +#define FLB_MP_GROUPS_KEEP_LEADING_END 1 +#define FLB_MP_GROUPS_KEEP_TRAILING_OPEN 2 + +int flb_mp_log_group_boundaries(const void *data, size_t bytes); +int flb_mp_log_group_starts(const void *data, size_t bytes); int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_size, char **out_buf, size_t *out_size); +int flb_mp_normalize_log_buffer_groups_msgpack_ex(const void *in_buf, size_t in_size, + char **out_buf, size_t *out_size, + int flags); int flb_mp_normalize_log_buffer_groups(const void *in_buf, size_t in_size, char **out_buf, size_t *out_size); int flb_mp_count_remaining(const void *data, size_t bytes, size_t *remaining_bytes); diff --git a/include/fluent-bit/flb_mp_chunk.h b/include/fluent-bit/flb_mp_chunk.h index 7f3ff751141..0ab5680e67b 100644 --- a/include/fluent-bit/flb_mp_chunk.h +++ b/include/fluent-bit/flb_mp_chunk.h @@ -30,6 +30,7 @@ /* Forward declaration to avoid circular dependencies */ struct flb_condition; +struct flb_mp_record_block; struct flb_mp_chunk_record { int modified; @@ -41,6 +42,9 @@ struct flb_mp_chunk_record { int owns_group_metadata; int owns_group_attributes; struct cfl_list _head; + + /* Pool block holding this record, NULL when allocated individually. */ + struct flb_mp_record_block *storage_block; }; struct flb_mp_chunk_cobj { @@ -56,6 +60,15 @@ struct flb_mp_chunk_cobj { /* Condition for filtering records during processing */ struct flb_condition *condition; + + /* Opt-in, bounded native value storage for the final serialization stage. */ + struct cfl_arena *value_pool; + size_t value_pool_bytes; + + /* Reference-counted wrapper blocks remain valid when records move. */ + int record_pool_enabled; + size_t record_pool_blocks; + struct flb_mp_record_block *record_pool; }; @@ -70,9 +83,17 @@ struct flb_mp_chunk_cobj *flb_mp_chunk_cobj_create(struct flb_log_event_encoder struct flb_log_event_decoder *log_decoder); int flb_mp_chunk_cobj_destroy(struct flb_mp_chunk_cobj *chunk_cobj); +/* Only enable when CFL values will be read and serialized, without mutation by + * a later native processor. The chunk owns the arena until destruction. */ +void flb_mp_chunk_cobj_value_pool_enable(struct flb_mp_chunk_cobj *chunk_cobj); +struct cfl_arena *flb_mp_chunk_cobj_value_pool_get(struct flb_mp_chunk_cobj *chunk_cobj, + size_t estimated_bytes); + +int flb_mp_chunk_cobj_encode_append(struct flb_mp_chunk_cobj *chunk_cobj); int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size); int flb_mp_chunk_cobj_count_log_records(struct flb_mp_chunk_cobj *chunk_cobj); int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj); +int flb_mp_chunk_cobj_normalize_groups_ex(struct flb_mp_chunk_cobj *chunk_cobj, int flags); diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index 6fce9a3bcd0..abaf431a68c 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -902,9 +902,10 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, normalized_buf = NULL; normalized_size = 0; - ret = flb_mp_normalize_log_buffer_groups_msgpack(p_buf, p_size, - &normalized_buf, - &normalized_size); + /* keep only the group markers that span chunks in the stored data */ + ret = flb_mp_normalize_log_buffer_groups_msgpack_ex( + p_buf, p_size, &normalized_buf, &normalized_size, + flb_mp_log_group_boundaries(evc->data, evc->size)); if (ret == 0) { if (p_buf != evc->data) { flb_free(p_buf); @@ -1151,6 +1152,8 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, return NULL; } + /* Capacity can exceed the serialized trace bytes after processing. */ + p_size = serialization_buffer_offset; out_flush->processed_event_chunk = flb_event_chunk_create( evc->type, 0, @@ -1251,6 +1254,8 @@ struct flb_output_flush *flb_output_flush_create(struct flb_task *task, return NULL; } + /* Capacity can exceed the serialized profile bytes after processing. */ + p_size = serialization_buffer_offset; out_flush->processed_event_chunk = flb_event_chunk_create( evc->type, 0, diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index 2a56531cbf5..8b47488442a 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -44,6 +44,15 @@ #define FLB_PROCESSOR_UNIT_NATIVE 0 #define FLB_PROCESSOR_UNIT_FILTER 1 +/* + * Plugin flags + * + * FLB_PROCESSOR_FLAG_LOG_WINDOWS: the logs callback only depends on each + * record and its group, so the processor chain can run it over large buffers + * in group aligned windows. + */ +#define FLB_PROCESSOR_FLAG_LOG_WINDOWS 1 + /* The current values mean the processor stack will * wait for 2 seconds at most in 50 millisecond increments @@ -148,7 +157,7 @@ struct flb_processor { }; struct flb_processor_plugin { - int flags; /* Flags (not available at the moment */ + int flags; /* FLB_PROCESSOR_FLAG_* capabilities */ char *name; /* Processor short name */ char *description; /* Description */ @@ -240,6 +249,12 @@ int flb_processor_run(struct flb_processor *proc, void **out_buf, size_t *out_size); +/* Carry known logical log counts through the pipeline. -1 means unknown. */ +int flb_processor_run_with_count(struct flb_processor *proc, size_t starting_stage, int type, + const char *tag, size_t tag_len, void *data, size_t data_size, + void **out_buf, size_t *out_size, + int input_records, int *output_records); + struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, int event_type, char *unit_name); diff --git a/lib/cfl/ARENA.md b/lib/cfl/ARENA.md index 3409f98900f..b41cdf2ecc4 100644 --- a/lib/cfl/ARENA.md +++ b/lib/cfl/ARENA.md @@ -259,8 +259,19 @@ Setting the limit to zero disables external-buffer caching. Reducing the limit immediately trims the cache. A larger limit can improve throughput for repeated large payloads but may increase retained memory and RSS. -The arena also maintains size classes for commonly-sized owned SDS values. -Callers do not need to select a class. +The arena also maintains size classes for owned SDS values up to 1 KiB. +Callers do not need to select a class. Classes below 128 bytes grow in +alignment-sized steps of the complete allocation, so short strings such as map +keys use at most one alignment unit more than they need; larger classes grow +by about 25 percent. A released string is recycled for the next string of the +same class. `cfl_sds_alloc()` reports the class capacity, which can exceed the +requested length. + +Variants, kvpairs, key/value lists, arrays, and SDS headers are aligned for the +pointers, sizes, 64-bit integers, and doubles they contain. This can be less +strict than the raw allocation alignment, which also covers `long double`, so +CFL objects pack more densely in arena chunks. Raw allocations keep the public +alignment guarantee. ## Memory statistics diff --git a/lib/cfl/CHANGELOG.md b/lib/cfl/CHANGELOG.md index 37b1f89347f..30e65835d8d 100644 --- a/lib/cfl/CHANGELOG.md +++ b/lib/cfl/CHANGELOG.md @@ -6,6 +6,15 @@ This file records the notable changes in each CFL release. - Added public request-lifetime arena allocation, duplication helpers, allocator callbacks, and optional bounded geometric chunk growth. +- Reduced arena memory for CFL object graphs: objects use their natural + alignment instead of the raw allocation alignment, and owned SDS strings use + finer size classes. Decoded log records use 27 to 31 percent less arena + memory. Owned SDS blocks are no longer zeroed before initialization. +- Added a bump-allocation fast path for the current arena chunk. +- Fixed the alignment of external arena allocations, which returned storage + at a misaligned offset for large owned SDS strings. +- Added the `cfl-benchmark-arena-records` benchmark for decoded record + batches. ## 1.0.0 - 2026-07-11 diff --git a/lib/cfl/benchmarks/CMakeLists.txt b/lib/cfl/benchmarks/CMakeLists.txt index f1148df43e8..0a7aefa05c5 100644 --- a/lib/cfl/benchmarks/CMakeLists.txt +++ b/lib/cfl/benchmarks/CMakeLists.txt @@ -3,3 +3,6 @@ target_link_libraries(cfl-benchmark-variant-arena cfl-static) add_executable(cfl-benchmark-variant-mutable variant_mutable.c) target_link_libraries(cfl-benchmark-variant-mutable cfl-static) + +add_executable(cfl-benchmark-arena-records records.c) +target_link_libraries(cfl-benchmark-arena-records cfl-static) diff --git a/lib/cfl/benchmarks/README.md b/lib/cfl/benchmarks/README.md index d4cb58f81fa..3697ad89293 100644 --- a/lib/cfl/benchmarks/README.md +++ b/lib/cfl/benchmarks/README.md @@ -88,3 +88,20 @@ document is still alive. Compare them with `arena_initial_used` and `arena_used`. `max_rss_kb` is process-level high-water RSS and can remain the same for small workloads because it is page-granular and includes executable, library, and allocator-retained pages. + +## Decoded record batches + +The records benchmark builds batches of small log records, each with short +keys, a nested map, an array, and string, integer, and double values, as +produced by decoding a buffer of telemetry records. Arena batches are reset +between iterations; heap batches destroy every record: + +```sh +build-bench/benchmarks/cfl-benchmark-arena-records arena 512 3000 +build-bench/benchmarks/cfl-benchmark-arena-records heap 512 3000 +``` + +Optional arguments select `referenced` or `owned` values, enable record edits +(`1` removes, inserts, and renames keys), and set the arena chunk size. The +tool reports nanoseconds per record and, for arenas, used and reserved bytes +per record. diff --git a/lib/cfl/benchmarks/records.c b/lib/cfl/benchmarks/records.c new file mode 100644 index 00000000000..4c2cd854728 --- /dev/null +++ b/lib/cfl/benchmarks/records.c @@ -0,0 +1,195 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* + * Decoded telemetry batch benchmark: every batch holds many small records + * with short keys, a nested map and an array, as produced by decoding a + * buffer of log records. Values can reference the source buffer or be owned + * copies, and records can be edited after construction. + */ + +#include +#include +#include +#include + +#if !defined(CFL_SYSTEM_WINDOWS) +#include +#endif + +#include + +struct batch_options { + size_t records; + int referenced; + int mutate; +}; + +static int insert_string(struct cfl_kvlist *list, char *key, char *value, + int referenced) +{ + return cfl_kvlist_insert_string_s(list, key, strlen(key), value, + strlen(value), referenced); +} + +static struct cfl_variant *build_record(struct cfl_arena *arena, size_t index, + struct batch_options *options) +{ + char message[64]; + struct cfl_kvlist *record; + struct cfl_kvlist *http; + struct cfl_array *tags; + struct cfl_variant *root; + + snprintf(message, sizeof(message), "request %zu completed", index); + + record = cfl_kvlist_create_in(arena); + http = cfl_kvlist_create_in(arena); + tags = cfl_array_create_in(arena, 2); + if (record == NULL || http == NULL || tags == NULL) { + return NULL; + } + + /* referenced values must outlive the record, keep them static */ + if (insert_string(record, "msg", options->referenced ? + "request completed" : message, + options->referenced) != 0 || + insert_string(record, "level", "info", options->referenced) != 0 || + insert_string(record, "user", "u-1234", options->referenced) != 0 || + insert_string(record, "password", "secret", options->referenced) != 0 || + insert_string(http, "path", "/api/v1/items", options->referenced) != 0 || + insert_string(http, "method", "GET", options->referenced) != 0 || + cfl_kvlist_insert_int64(http, "status", 200) != 0 || + cfl_kvlist_insert_kvlist(record, "http", http) != 0 || + cfl_kvlist_insert_double(record, "duration_ms", 12.5) != 0 || + cfl_array_append_string_s(tags, "edge", 4, options->referenced) != 0 || + cfl_array_append_string_s(tags, "eu-west-1", 9, + options->referenced) != 0 || + cfl_kvlist_insert_array(record, "tags", tags) != 0) { + return NULL; + } + + if (options->mutate) { + if (cfl_kvlist_remove(record, "password") != CFL_TRUE || + cfl_kvlist_insert_int64(record, "x", 1) != 0 || + cfl_kvlist_rename_s(record, "user", 4, "user.id", 7) != 0) { + return NULL; + } + } + + root = cfl_variant_create_from_kvlist_in(arena, record); + return root; +} + +static int run_batch(struct cfl_arena *arena, struct cfl_variant **roots, + struct batch_options *options) +{ + size_t index; + int result; + + result = 0; + for (index = 0; index < options->records; index++) { + roots[index] = build_record(arena, index, options); + if (roots[index] == NULL) { + result = -1; + break; + } + } + + /* arena records are released by the reset, heap ones one by one */ + if (arena == NULL) { + for (index = 0; index < options->records; index++) { + if (roots[index] != NULL) { + cfl_variant_destroy(roots[index]); + } + } + } + + return result; +} + +int main(int argc, char **argv) +{ + const char *mode; + size_t batches; + size_t batch; + size_t chunk_size; + size_t reserved; + size_t used; + uint64_t start; + uint64_t elapsed; + struct cfl_arena *arena; + struct cfl_variant **roots; + struct batch_options options; +#if !defined(CFL_SYSTEM_WINDOWS) + struct rusage usage; +#endif + + mode = argc > 1 ? argv[1] : "arena"; + options.records = argc > 2 ? strtoull(argv[2], NULL, 10) : 512; + batches = argc > 3 ? strtoull(argv[3], NULL, 10) : 2000; + options.referenced = argc > 4 ? strcmp(argv[4], "owned") != 0 : CFL_TRUE; + options.mutate = argc > 5 ? atoi(argv[5]) : 0; + chunk_size = argc > 6 ? strtoull(argv[6], NULL, 10) : 16384; + + if ((strcmp(mode, "arena") != 0 && strcmp(mode, "heap") != 0) || + options.records == 0) { + fprintf(stderr, + "usage: %s arena|heap [records] [batches] " + "[referenced|owned] [mutate] [chunk-size]\n", argv[0]); + return EXIT_FAILURE; + } + + roots = calloc(options.records, sizeof(struct cfl_variant *)); + if (roots == NULL) { + return EXIT_FAILURE; + } + + arena = NULL; + if (strcmp(mode, "arena") == 0) { + arena = cfl_arena_create(chunk_size); + if (arena == NULL) { + free(roots); + return EXIT_FAILURE; + } + } + + reserved = 0; + used = 0; + start = cfl_time_now(); + for (batch = 0; batch < batches; batch++) { + if (run_batch(arena, roots, &options) != 0) { + fprintf(stderr, "batch construction failed\n"); + cfl_arena_destroy(arena); + free(roots); + return EXIT_FAILURE; + } + if (arena != NULL) { + reserved = cfl_arena_bytes_reserved(arena); + used = cfl_arena_bytes_used(arena); + cfl_arena_reset(arena); + } + } + elapsed = cfl_time_now() - start; + + printf("mode=%s records=%zu batches=%zu values=%s mutate=%d " + "elapsed_ns=%llu ns_per_record=%.2f", + mode, options.records, batches, + options.referenced ? "referenced" : "owned", options.mutate, + (unsigned long long) elapsed, + (double) elapsed / (double) (batches * options.records)); + if (arena != NULL) { + printf(" arena_reserved=%zu arena_used=%zu used_per_record=%.1f " + "reserved_per_record=%.1f", reserved, used, + (double) used / (double) options.records, + (double) reserved / (double) options.records); + } +#if !defined(CFL_SYSTEM_WINDOWS) + getrusage(RUSAGE_SELF, &usage); + printf(" max_rss_kb=%ld", usage.ru_maxrss); +#endif + putchar('\n'); + + cfl_arena_destroy(arena); + free(roots); + return EXIT_SUCCESS; +} diff --git a/lib/cfl/src/cfl_arena.c b/lib/cfl/src/cfl_arena.c index 62fa165fddd..a3c80e80a94 100644 --- a/lib/cfl/src/cfl_arena.c +++ b/lib/cfl/src/cfl_arena.c @@ -5,11 +5,15 @@ #include #include +#include +#include +#include #include "cfl_arena_internal.h" #define CFL_ARENA_DEFAULT_CHUNK_SIZE 8192 -#define CFL_ARENA_SDS_CLASS_COUNT 6 +#define CFL_ARENA_SDS_SMALL_PAYLOAD 128 +#define CFL_ARENA_SDS_MEDIUM_CLASS_COUNT 10 #define CFL_ARENA_EXTERNAL_CLASS_COUNT 24 #define CFL_ARENA_EXTERNAL_EXACT_CLASS UINT8_MAX @@ -24,6 +28,72 @@ struct cfl_arena_alignment_probe { union cfl_arena_max_align value; }; +/* + * CFL objects only hold pointers, sizes, integers and doubles, so they are + * packed with the strictest of those alignments instead of the raw allocation + * alignment, which can be larger because of long double. + */ +union cfl_arena_object_align { + void *pointer_value; + size_t size_value; + uint64_t uint64_value; + double double_value; +}; + +struct cfl_arena_object_alignment_probe { + char byte; + union cfl_arena_object_align value; +}; + +#define CFL_ARENA_MAX_ALIGNMENT \ + offsetof(struct cfl_arena_alignment_probe, value) +#define CFL_ARENA_OBJECT_ALIGNMENT \ + offsetof(struct cfl_arena_object_alignment_probe, value) + +/* small SDS classes cover payloads below 128 bytes in alignment steps */ +#define CFL_ARENA_SDS_SMALL_CLASS_COUNT \ + (CFL_ARENA_SDS_SMALL_PAYLOAD / CFL_ARENA_OBJECT_ALIGNMENT) +#define CFL_ARENA_SDS_CLASS_COUNT \ + (CFL_ARENA_SDS_SMALL_CLASS_COUNT + CFL_ARENA_SDS_MEDIUM_CLASS_COUNT) + +struct cfl_arena_variant_probe { + char byte; + struct cfl_variant value; +}; + +struct cfl_arena_kvpair_probe { + char byte; + struct cfl_kvpair value; +}; + +struct cfl_arena_kvlist_probe { + char byte; + struct cfl_kvlist value; +}; + +struct cfl_arena_array_probe { + char byte; + struct cfl_array value; +}; + +/* compile time checks: every object type fits the object alignment */ +typedef char cfl_arena_check_object_alignment[ + (CFL_ARENA_MAX_ALIGNMENT % CFL_ARENA_OBJECT_ALIGNMENT == 0 && + (CFL_ARENA_OBJECT_ALIGNMENT & (CFL_ARENA_OBJECT_ALIGNMENT - 1)) == 0 && + CFL_ARENA_SDS_CLASS_COUNT < UINT8_MAX) ? 1 : -1]; +typedef char cfl_arena_check_variant_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_variant_probe, value) == 0) ? 1 : -1]; +typedef char cfl_arena_check_kvpair_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_kvpair_probe, value) == 0) ? 1 : -1]; +typedef char cfl_arena_check_kvlist_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_kvlist_probe, value) == 0) ? 1 : -1]; +typedef char cfl_arena_check_array_alignment[ + (CFL_ARENA_OBJECT_ALIGNMENT % + offsetof(struct cfl_arena_array_probe, value) == 0) ? 1 : -1]; + struct cfl_arena_chunk { struct cfl_arena_chunk *next; size_t capacity; @@ -37,6 +107,7 @@ struct cfl_arena_external { struct cfl_arena_external *previous; size_t size; uint8_t allocation_class; + union cfl_arena_max_align alignment; unsigned char data[]; }; @@ -276,24 +347,20 @@ void cfl_arena_reset(struct cfl_arena *arena) memset(arena->free_sds, 0, sizeof(arena->free_sds)); } -void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) +/* walks the remaining chunks, then reserves a new one */ +static void *arena_allocate_slow(struct cfl_arena *arena, size_t size, + size_t alignment) { struct cfl_arena_chunk *chunk; - size_t alignment; size_t offset; size_t remainder; size_t capacity; void *result; - if (arena == NULL || size == 0) { - return NULL; - } - - alignment = offsetof(struct cfl_arena_alignment_probe, value); chunk = arena->current; while (chunk != NULL) { offset = chunk->used; - remainder = offset % alignment; + remainder = offset & (alignment - 1); if (remainder != 0) { if (offset > SIZE_MAX - (alignment - remainder)) { return NULL; @@ -346,6 +413,62 @@ void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) return chunk->data; } +/* bump allocation from the current chunk, the common case */ +static inline void *arena_allocate(struct cfl_arena *arena, size_t size, + size_t alignment) +{ + struct cfl_arena_chunk *chunk; + size_t offset; + void *result; + + if (arena == NULL || size == 0) { + return NULL; + } + + chunk = arena->current; + if (chunk != NULL) { + offset = (chunk->used + alignment - 1) & ~(alignment - 1); + if (offset >= chunk->used && offset <= chunk->capacity && + size <= chunk->capacity - offset) { + result = &chunk->data[offset]; + chunk->used = offset + size; + arena->bytes_used += size; + return result; + } + } + + return arena_allocate_slow(arena, size, alignment); +} + +void *cfl_arena_malloc(struct cfl_arena *arena, size_t size) +{ + return arena_allocate(arena, size, CFL_ARENA_MAX_ALIGNMENT); +} + +void *cfl_arena_alloc_object(struct cfl_arena *arena, size_t size) +{ + return arena_allocate(arena, size, CFL_ARENA_OBJECT_ALIGNMENT); +} + +void *cfl_arena_calloc_object(struct cfl_arena *arena, + size_t count, size_t size) +{ + void *result; + size_t total; + + if (count != 0 && size > SIZE_MAX / count) { + return NULL; + } + + total = count * size; + result = arena_allocate(arena, total, CFL_ARENA_OBJECT_ALIGNMENT); + if (result != NULL) { + memset(result, 0, total); + } + + return result; +} + void *cfl_arena_alloc(struct cfl_arena *arena, size_t size) { return cfl_arena_malloc(arena, size); @@ -604,7 +727,7 @@ void cfl_arena_free_external(struct cfl_arena *arena, } static void *arena_reusable_alloc(struct cfl_arena *arena, - void **free_list, size_t size) + void **free_list, size_t size, int zero) { void *result; @@ -616,11 +739,19 @@ static void *arena_reusable_alloc(struct cfl_arena *arena, result = *free_list; *free_list = *((void **) result); arena->bytes_used += size; + } + else { + result = arena_allocate(arena, size, CFL_ARENA_OBJECT_ALIGNMENT); + if (result == NULL) { + return NULL; + } + } + + if (zero) { memset(result, 0, size); - return result; } - return cfl_arena_calloc(arena, 1, size); + return result; } static void arena_reusable_free(struct cfl_arena *arena, @@ -638,7 +769,7 @@ static void arena_reusable_free(struct cfl_arena *arena, void *cfl_arena_alloc_variant(struct cfl_arena *arena, size_t size) { - return arena_reusable_alloc(arena, &arena->free_variants, size); + return arena_reusable_alloc(arena, &arena->free_variants, size, CFL_TRUE); } void cfl_arena_free_variant(struct cfl_arena *arena, @@ -650,7 +781,7 @@ void cfl_arena_free_variant(struct cfl_arena *arena, void *cfl_arena_alloc_kvpair(struct cfl_arena *arena, size_t size) { - return arena_reusable_alloc(arena, &arena->free_kvpairs, size); + return arena_reusable_alloc(arena, &arena->free_kvpairs, size, CFL_TRUE); } void cfl_arena_free_kvpair(struct cfl_arena *arena, @@ -659,32 +790,80 @@ void cfl_arena_free_kvpair(struct cfl_arena *arena, arena_reusable_free(arena, &arena->free_kvpairs, pointer, size); } +/* + * Owned SDS strings are recycled through size classes. Small classes grow in + * object alignment steps of the complete allocation up to about 128 bytes of + * payload, so short strings such as map keys waste less than one alignment + * unit. Medium classes grow by roughly 25 percent up to the largest pooled + * payload. + */ +static int arena_sds_class(size_t payload_size, size_t overhead_size, + size_t *index, size_t *payload_capacity) +{ + static const size_t medium_payloads[CFL_ARENA_SDS_MEDIUM_CLASS_COUNT] = { + 160, 200, 248, 312, 392, 488, 608, 760, 952, CFL_ARENA_SDS_MAX_PAYLOAD + }; + size_t alignment; + size_t smallest; + size_t total; + size_t class_index; + + alignment = CFL_ARENA_OBJECT_ALIGNMENT; + if (payload_size > CFL_ARENA_SDS_MAX_PAYLOAD || + overhead_size > SIZE_MAX - CFL_ARENA_SDS_MAX_PAYLOAD - 1 - alignment) { + return -1; + } + + /* smallest allocation holds the overhead, the terminator and one byte */ + smallest = (overhead_size + 2 + alignment - 1) & ~(alignment - 1); + total = (overhead_size + payload_size + 1 + alignment - 1) & + ~(alignment - 1); + if (total < smallest) { + total = smallest; + } + + if (total <= smallest + (CFL_ARENA_SDS_SMALL_CLASS_COUNT - 1) * alignment) { + *index = (total - smallest) / alignment; + *payload_capacity = total - overhead_size - 1; + return 0; + } + + for (class_index = 0; class_index < CFL_ARENA_SDS_MEDIUM_CLASS_COUNT; + class_index++) { + total = (overhead_size + medium_payloads[class_index] + 1 + + alignment - 1) & ~(alignment - 1); + if (payload_size <= total - overhead_size - 1) { + *index = CFL_ARENA_SDS_SMALL_CLASS_COUNT + class_index; + *payload_capacity = total - overhead_size - 1; + return 0; + } + } + + return -1; +} + void *cfl_arena_alloc_sds(struct cfl_arena *arena, size_t payload_size, size_t overhead_size, uint8_t *allocation_class, size_t *payload_capacity) { - static const size_t class_sizes[CFL_ARENA_SDS_CLASS_COUNT] = { - 32, 64, 128, 256, 512, 1024 - }; size_t index; if (arena == NULL || allocation_class == NULL || payload_capacity == NULL) { return NULL; } - for (index = 0; index < CFL_ARENA_SDS_CLASS_COUNT; index++) { - if (payload_size <= class_sizes[index]) { - *allocation_class = (uint8_t) (index + 1); - *payload_capacity = class_sizes[index]; - return arena_reusable_alloc(arena, &arena->free_sds[index], - overhead_size + class_sizes[index] + 1); - } + if (arena_sds_class(payload_size, overhead_size, &index, + payload_capacity) != 0) { + *allocation_class = 0; + *payload_capacity = payload_size; + return NULL; } - *allocation_class = 0; - *payload_capacity = payload_size; - return NULL; + /* the SDS layer initializes the headers and the payload it uses */ + *allocation_class = (uint8_t) (index + 1); + return arena_reusable_alloc(arena, &arena->free_sds[index], + overhead_size + *payload_capacity + 1, CFL_FALSE); } void cfl_arena_free_sds(struct cfl_arena *arena, diff --git a/lib/cfl/src/cfl_arena_internal.h b/lib/cfl/src/cfl_arena_internal.h index e41fe3a943b..5959a17ece1 100644 --- a/lib/cfl/src/cfl_arena_internal.h +++ b/lib/cfl/src/cfl_arena_internal.h @@ -2,10 +2,23 @@ #define CFL_ARENA_INTERNAL_H #include +#include #include +/* largest owned SDS payload recycled through arena size classes */ +#define CFL_ARENA_SDS_MAX_PAYLOAD 1024 + void *cfl_arena_alloc(struct cfl_arena *arena, size_t size); + +/* + * CFL object allocations: aligned for the pointers, sizes, integers and + * doubles held by variants, kvpairs, kvlists, arrays and SDS headers, which + * can be less strict than the public raw allocation alignment. + */ +void *cfl_arena_alloc_object(struct cfl_arena *arena, size_t size); +void *cfl_arena_calloc_object(struct cfl_arena *arena, + size_t count, size_t size); void *cfl_arena_alloc_external(struct cfl_arena *arena, size_t size); void cfl_arena_free_external(struct cfl_arena *arena, diff --git a/lib/cfl/src/cfl_array.c b/lib/cfl/src/cfl_array.c index bac55ab67a9..d7b0a99c271 100644 --- a/lib/cfl/src/cfl_array.c +++ b/lib/cfl/src/cfl_array.c @@ -49,7 +49,7 @@ struct cfl_array *cfl_array_create_in(struct cfl_arena *arena, array = malloc(sizeof(struct cfl_array)); } else { - array = cfl_arena_malloc(arena, sizeof(struct cfl_array)); + array = cfl_arena_alloc_object(arena, sizeof(struct cfl_array)); } if (array == NULL) { cfl_errno(); @@ -64,8 +64,8 @@ struct cfl_array *cfl_array_create_in(struct cfl_arena *arena, array->entries = calloc(alloc_count, sizeof(void *)); } else { - array->entries = cfl_arena_calloc(arena, alloc_count, - sizeof(void *)); + array->entries = cfl_arena_calloc_object(arena, alloc_count, + sizeof(void *)); } if (array->entries == NULL) { cfl_errno(); @@ -222,7 +222,7 @@ int cfl_array_append(struct cfl_array *array, tmp = realloc(array->entries, new_size); } else { - tmp = cfl_arena_malloc(array->arena, new_size); + tmp = cfl_arena_alloc_object(array->arena, new_size); if (tmp != NULL) { memcpy(tmp, array->entries, array->entry_count * sizeof(void *)); diff --git a/lib/cfl/src/cfl_kvlist.c b/lib/cfl/src/cfl_kvlist.c index 204ab7744ea..3993e48381a 100644 --- a/lib/cfl/src/cfl_kvlist.c +++ b/lib/cfl/src/cfl_kvlist.c @@ -99,7 +99,7 @@ struct cfl_kvlist *cfl_kvlist_create_in(struct cfl_arena *arena) list = malloc(sizeof(struct cfl_kvlist)); } else { - list = cfl_arena_malloc(arena, sizeof(struct cfl_kvlist)); + list = cfl_arena_alloc_object(arena, sizeof(struct cfl_kvlist)); } if (list == NULL) { cfl_report_runtime_error(); diff --git a/lib/cfl/src/cfl_sds.c b/lib/cfl/src/cfl_sds.c index afe89c2a937..13e39234259 100644 --- a/lib/cfl/src/cfl_sds.c +++ b/lib/cfl/src/cfl_sds.c @@ -82,8 +82,9 @@ static cfl_sds_t sds_alloc(struct cfl_arena *arena, size_t size) if (arena == NULL) { buf = malloc(allocation_size); } - else if (size <= 1024) { - external_threshold = cfl_arena_large_object_threshold(arena); + else if (size <= CFL_ARENA_SDS_MAX_PAYLOAD) { + /* pooled strings never use external storage */ + external_threshold = SIZE_MAX; buf = cfl_arena_alloc_sds(arena, size, CFL_SDS_HEADER_SIZE + sizeof(struct cfl_sds_arena_header), @@ -99,7 +100,7 @@ static cfl_sds_t sds_alloc(struct cfl_arena *arena, size_t size) buf = cfl_arena_alloc_external(arena, allocation_size); } else { - buf = cfl_arena_malloc(arena, allocation_size); + buf = cfl_arena_alloc_object(arena, allocation_size); } } if (!buf) { diff --git a/lib/cfl/tests/arena.c b/lib/cfl/tests/arena.c index 494fa635088..77e4969567b 100644 --- a/lib/cfl/tests/arena.c +++ b/lib/cfl/tests/arena.c @@ -17,6 +17,56 @@ struct test_alignment_probe { union test_max_align value; }; +/* CFL objects hold pointers, sizes, integers and doubles */ +union test_object_align { + void *pointer_value; + size_t size_value; + uint64_t uint64_value; + double double_value; +}; + +struct test_object_alignment_probe { + char byte; + union test_object_align value; +}; + +struct test_variant_probe { + char byte; + struct cfl_variant value; +}; + +struct test_kvpair_probe { + char byte; + struct cfl_kvpair value; +}; + +struct test_kvlist_probe { + char byte; + struct cfl_kvlist value; +}; + +struct test_array_probe { + char byte; + struct cfl_array value; +}; + +#define TEST_MAX_ALIGNMENT offsetof(struct test_alignment_probe, value) +#define TEST_OBJECT_ALIGNMENT offsetof(struct test_object_alignment_probe, value) + +/* owned SDS payloads up to this size are recycled through size classes */ +#define TEST_SDS_POOLED_PAYLOAD 1024 + +static int is_aligned(const void *pointer, size_t alignment) +{ + return ((uintptr_t) pointer % alignment) == 0; +} + +static uint32_t test_random(uint32_t *state) +{ + *state = (*state * 1103515245u) + 12345u; + return (*state >> 8) & 0xffffff; +} + struct test_allocator_context { size_t allocation_count; size_t free_count; @@ -344,6 +394,7 @@ static void reclaim_large_string_on_remove(void) value = cfl_kvlist_fetch(list, "body"); TEST_CHECK(value != NULL); first_buffer = value->data.as_string; + TEST_CHECK(is_aligned(CFL_SDS_HEADER(first_buffer), TEST_OBJECT_ALIGNMENT)); before_remove = cfl_arena_bytes_used(arena); reserved_before_remove = cfl_arena_bytes_reserved(arena); TEST_CHECK(cfl_kvlist_remove(list, "body") == CFL_TRUE); @@ -428,13 +479,15 @@ static void create_like_and_rename(void) static void reuse_sds_size_classes(void) { - static const size_t sizes[] = {1, 32, 33, 64, 65, 128, - 129, 256, 257, 512, 513, 1024}; + static const size_t sizes[] = {0, 1, 7, 8, 15, 16, 31, 32, 33, 64, 65, + 127, 128, 129, 256, 257, 512, 513, 1023, + 1024}; struct cfl_arena *arena; cfl_sds_t first; cfl_sds_t second; cfl_sds_t grown; char payload[1024]; + size_t capacity; size_t index; memset(payload, 's', sizeof(payload)); @@ -444,28 +497,565 @@ static void reuse_sds_size_classes(void) for (index = 0; index < sizeof(sizes) / sizeof(sizes[0]); index++) { first = cfl_sds_create_len_in(arena, payload, (int) sizes[index]); TEST_CHECK(first != NULL); + TEST_CHECK(cfl_sds_alloc(first) >= sizes[index]); cfl_sds_destroy(first); second = cfl_sds_create_len_in(arena, payload, (int) sizes[index]); TEST_CHECK(second == first); cfl_sds_destroy(second); } + /* growing past the capacity moves to the next class, keeping the data */ first = cfl_sds_create_len_in(arena, payload, 31); TEST_CHECK(first != NULL); - TEST_CHECK(cfl_sds_alloc(first) == 32); - grown = cfl_sds_increase(first, 1); + capacity = cfl_sds_alloc(first); + TEST_CHECK(capacity >= 31); + grown = cfl_sds_increase(first, capacity - 31 + 1); TEST_CHECK(grown != NULL); - TEST_CHECK(cfl_sds_alloc(grown) == 64); + TEST_CHECK(grown != first); + TEST_CHECK(cfl_sds_alloc(grown) > capacity); TEST_CHECK(cfl_sds_len(grown) == 31); TEST_CHECK(memcmp(grown, payload, 31) == 0); + TEST_CHECK(grown[31] == '\0'); - second = cfl_sds_create_len_in(arena, payload, 32); + /* the released slot serves the next string of the same class */ + second = cfl_sds_create_len_in(arena, payload, 31); TEST_CHECK(second == first); cfl_sds_destroy(second); cfl_sds_destroy(grown); cfl_arena_destroy(arena); } +static void sds_class_boundaries(void) +{ + struct cfl_arena *arena; + cfl_sds_t value; + cfl_sds_t again; + char payload[TEST_SDS_POOLED_PAYLOAD + 64]; + size_t previous_capacity; + size_t capacity; + size_t used; + size_t size; + + memset(payload, 'b', sizeof(payload)); + arena = cfl_arena_create(4096); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + previous_capacity = 0; + for (size = 0; size <= TEST_SDS_POOLED_PAYLOAD + 32; size++) { + used = cfl_arena_bytes_used(arena); + value = cfl_sds_create_len_in(arena, payload, (int) size); + TEST_CHECK(value != NULL); + if (value == NULL) { + break; + } + + capacity = cfl_sds_alloc(value); + TEST_CHECK(cfl_sds_len(value) == size); + TEST_CHECK(capacity >= size); + TEST_CHECK(cfl_sds_avail(value) == capacity - size); + TEST_CHECK(value[size] == '\0'); + TEST_CHECK(memcmp(value, payload, size) == 0); + TEST_CHECK(is_aligned(CFL_SDS_HEADER(value), TEST_OBJECT_ALIGNMENT)); + + if (size <= TEST_SDS_POOLED_PAYLOAD) { + /* class capacities never shrink as the payload grows */ + TEST_CHECK(capacity >= previous_capacity); + previous_capacity = capacity; + + /* short strings are sized within one alignment unit */ + if (size < 128) { + TEST_CHECK(capacity - size < TEST_OBJECT_ALIGNMENT); + } + else { + TEST_CHECK(capacity - size <= size / 4 + TEST_OBJECT_ALIGNMENT); + } + } + else { + TEST_CHECK(capacity == size); + } + + cfl_sds_destroy(value); + + /* larger strings below the external threshold live until reset */ + if (size <= TEST_SDS_POOLED_PAYLOAD) { + TEST_CHECK(cfl_arena_bytes_used(arena) == used); + again = cfl_sds_create_len_in(arena, payload, (int) size); + TEST_CHECK(again == value); + cfl_sds_destroy(again); + } + } + + cfl_arena_destroy(arena); +} + +static void sds_capacity_is_writable(void) +{ + struct cfl_arena *arena; + cfl_sds_t values[256]; + char payload[2048]; + size_t capacity; + size_t index; + size_t offset; + int intact; + + memset(payload, 'p', sizeof(payload)); + arena = cfl_arena_create(1024); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* fill every string to its reported capacity, neighbours must survive */ + for (index = 0; index < 256; index++) { + values[index] = cfl_sds_create_len_in(arena, payload, + (int) ((index * 37) % 1500)); + TEST_CHECK(values[index] != NULL); + if (values[index] == NULL) { + cfl_arena_destroy(arena); + return; + } + capacity = cfl_sds_alloc(values[index]); + memset(values[index], (int) ('A' + (index % 26)), capacity); + cfl_sds_set_len(values[index], capacity); + TEST_CHECK(values[index][capacity] == '\0'); + } + + for (index = 0; index < 256; index++) { + capacity = cfl_sds_alloc(values[index]); + TEST_CHECK(cfl_sds_len(values[index]) == capacity); + intact = 1; + for (offset = 0; offset < capacity; offset++) { + if (values[index][offset] != (char) ('A' + (index % 26))) { + intact = 0; + break; + } + } + TEST_CHECK(intact == 1); + TEST_CHECK(values[index][capacity] == '\0'); + TEST_MSG("string %zu with capacity %zu", index, capacity); + } + + for (index = 0; index < 256; index += 2) { + cfl_sds_destroy(values[index]); + } + for (index = 1; index < 256; index += 2) { + TEST_CHECK(values[index][0] == (char) ('A' + (index % 26))); + cfl_sds_destroy(values[index]); + } + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + + cfl_arena_destroy(arena); +} + +static void sds_grow_across_classes(void) +{ + struct cfl_arena *arena; + cfl_sds_t value; + cfl_sds_t result; + char expected[3001]; + size_t index; + + arena = cfl_arena_create_ex(2048, 1500); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* small classes, medium classes, chunk storage and external storage */ + value = cfl_sds_create_len_in(arena, "", 0); + TEST_CHECK(value != NULL); + for (index = 0; index < 3000 && value != NULL; index++) { + expected[index] = (char) ('a' + (index % 26)); + result = cfl_sds_cat(value, &expected[index], 1); + TEST_CHECK(result != NULL); + if (result == NULL) { + break; + } + value = result; + TEST_CHECK(is_aligned(CFL_SDS_HEADER(value), TEST_OBJECT_ALIGNMENT)); + TEST_CHECK(cfl_sds_len(value) == index + 1); + TEST_CHECK(cfl_sds_alloc(value) >= index + 1); + TEST_CHECK(value[index + 1] == '\0'); + } + expected[3000] = '\0'; + TEST_CHECK(value != NULL && strcmp(value, expected) == 0); + + /* self append and printf keep working on arena storage */ + result = cfl_sds_cat(value, value, 10); + TEST_CHECK(result != NULL); + if (result != NULL) { + value = result; + TEST_CHECK(cfl_sds_len(value) == 3010); + TEST_CHECK(memcmp(value + 3000, expected, 10) == 0); + } + result = cfl_sds_printf(&value, "-%d", 42); + TEST_CHECK(result != NULL); + TEST_CHECK(cfl_sds_len(value) == 3013); + TEST_CHECK(strcmp(value + 3010, "-42") == 0); + + cfl_sds_destroy(value); + cfl_arena_destroy(arena); +} + +static void object_alignment(void) +{ + struct cfl_arena *arena; + struct cfl_variant *variant; + struct cfl_kvlist *list; + struct cfl_kvpair *pair; + struct cfl_array *array; + struct cfl_variant *entry; + struct cfl_list *head; + char key[8]; + size_t index; + + arena = cfl_arena_create(512); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + list = cfl_kvlist_create_in(arena); + array = cfl_array_create_in(arena, 1); + TEST_CHECK(list != NULL && array != NULL); + if (list == NULL || array == NULL) { + cfl_arena_destroy(arena); + return; + } + TEST_CHECK(cfl_array_resizable(array, CFL_TRUE) == 0); + + /* odd sized keys and payloads shift the bump pointer between objects */ + for (index = 0; index < 64; index++) { + snprintf(key, sizeof(key), "k%zu", index); + TEST_CHECK(cfl_kvlist_insert_string_s(list, key, strlen(key), "abc", + index % 4, CFL_FALSE) == 0); + TEST_CHECK(cfl_kvlist_insert_double(list, "", (double) index) == 0); + TEST_CHECK(cfl_array_append_uint64(array, index) == 0); + + variant = cfl_variant_create_from_int64_in(arena, (int64_t) index); + TEST_CHECK(variant != NULL); + TEST_CHECK(is_aligned(variant, TEST_OBJECT_ALIGNMENT)); + TEST_CHECK(is_aligned(variant, + offsetof(struct test_variant_probe, value))); + cfl_variant_destroy(variant); + } + + TEST_CHECK(is_aligned(list, offsetof(struct test_kvlist_probe, value))); + TEST_CHECK(is_aligned(array, offsetof(struct test_array_probe, value))); + TEST_CHECK(is_aligned(array->entries, sizeof(void *))); + + cfl_list_foreach(head, &list->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + TEST_CHECK(is_aligned(pair, offsetof(struct test_kvpair_probe, value))); + TEST_CHECK(is_aligned(pair->val, + offsetof(struct test_variant_probe, value))); + TEST_CHECK(is_aligned(CFL_SDS_HEADER(pair->key), + TEST_OBJECT_ALIGNMENT)); + } + + for (index = 0; index < cfl_array_size(array); index++) { + entry = cfl_array_fetch_by_index(array, index); + TEST_CHECK(entry != NULL); + TEST_CHECK(is_aligned(entry, offsetof(struct test_variant_probe, value))); + TEST_CHECK(entry->data.as_uint64 == index); + } + + cfl_kvlist_destroy(list); + cfl_array_destroy(array); + cfl_arena_destroy(arena); +} + +static void raw_alignment_after_objects(void) +{ + struct cfl_arena *arena; + struct cfl_variant *variant; + cfl_sds_t key; + void *pointer; + size_t index; + + arena = cfl_arena_create(256); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* raw allocations keep the public alignment between packed objects */ + for (index = 0; index < 64; index++) { + variant = cfl_variant_create_from_bool_in(arena, CFL_TRUE); + TEST_CHECK(variant != NULL); + key = cfl_sds_create_len_in(arena, "key", (int) (index % 5)); + TEST_CHECK(key != NULL); + + pointer = cfl_arena_malloc(arena, 1 + (index % 3)); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + pointer = cfl_arena_calloc(arena, 3, 1); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + pointer = cfl_arena_memdup(arena, "abc", 3); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + pointer = cfl_arena_strndup(arena, "abc", 2); + TEST_CHECK(pointer != NULL && is_aligned(pointer, TEST_MAX_ALIGNMENT)); + } + + cfl_arena_destroy(arena); +} + +static void compact_short_keys(void) +{ + struct cfl_arena *arena; + struct cfl_kvlist *record; + struct cfl_variant *root; + static char *keys[] = {"msg", "user", "path", "status", "level", "ts"}; + size_t record_count; + size_t index; + size_t key_index; + size_t limit; + size_t reserved; + size_t object; + + record_count = 256; + arena = cfl_arena_create(16384); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + /* worst case: every object padded by one alignment unit */ + object = TEST_OBJECT_ALIGNMENT; + limit = sizeof(struct cfl_kvlist) + object + sizeof(struct cfl_variant) + object; + for (key_index = 0; key_index < 6; key_index++) { + limit += sizeof(struct cfl_kvpair) + object + + sizeof(struct cfl_variant) + object + + 2 * sizeof(void *) + CFL_SDS_HEADER_SIZE + + strlen(keys[key_index]) + 1 + object; + } + + for (index = 0; index < record_count; index++) { + record = cfl_kvlist_create_in(arena); + TEST_CHECK(record != NULL); + if (record == NULL) { + break; + } + for (key_index = 0; key_index < 6; key_index++) { + TEST_CHECK(cfl_kvlist_insert_int64(record, keys[key_index], + (int64_t) index) == 0); + } + root = cfl_variant_create_from_kvlist_in(arena, record); + TEST_CHECK(root != NULL); + } + + /* allow one partially used chunk plus chunk headers */ + reserved = cfl_arena_bytes_reserved(arena); + TEST_CHECK(reserved <= record_count * limit + 16384 + 256); + TEST_MSG("reserved %zu limit %zu", reserved, record_count * limit + 16384 + 256); + + cfl_arena_destroy(arena); +} + +static void reset_reuses_storage_without_aliasing(void) +{ + struct cfl_arena *arena; + cfl_sds_t first; + cfl_sds_t second; + struct cfl_variant *variant; + struct cfl_variant *other; + size_t reserved; + int round; + + arena = cfl_arena_create(1024); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + return; + } + + for (round = 0; round < 8; round++) { + /* released slots must not survive the reset in the free lists */ + first = cfl_sds_create_len_in(arena, "abc", 3); + variant = cfl_variant_create_from_int64_in(arena, round); + TEST_CHECK(first != NULL && variant != NULL); + cfl_sds_destroy(first); + cfl_variant_destroy(variant); + reserved = cfl_arena_bytes_reserved(arena); + cfl_arena_reset(arena); + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + TEST_CHECK(cfl_arena_bytes_reserved(arena) == reserved); + + first = cfl_sds_create_len_in(arena, "abc", 3); + second = cfl_sds_create_len_in(arena, "xyz", 3); + variant = cfl_variant_create_from_int64_in(arena, 1); + other = cfl_variant_create_from_int64_in(arena, 2); + TEST_CHECK(first != NULL && second != NULL); + TEST_CHECK(variant != NULL && other != NULL); + TEST_CHECK(first != second && variant != other); + TEST_CHECK(strcmp(first, "abc") == 0 && strcmp(second, "xyz") == 0); + TEST_CHECK(variant->data.as_int64 == 1 && other->data.as_int64 == 2); + cfl_arena_reset(arena); + } + + cfl_arena_destroy(arena); +} + +static void heap_storage_unchanged(void) +{ + cfl_sds_t value; + struct cfl_kvlist *list; + struct cfl_kvpair *pair; + + /* heap strings keep their exact capacities */ + value = cfl_sds_create_len("abc", 3); + TEST_CHECK(value != NULL); + TEST_CHECK(cfl_sds_alloc(value) == 3); + cfl_sds_destroy(value); + + value = cfl_sds_create_size(100); + TEST_CHECK(value != NULL); + TEST_CHECK(cfl_sds_alloc(value) == 100); + TEST_CHECK(cfl_sds_len(value) == 0); + cfl_sds_destroy(value); + + list = cfl_kvlist_create(); + TEST_CHECK(list != NULL); + if (list == NULL) { + return; + } + TEST_CHECK(cfl_kvlist_insert_int64(list, "status", 200) == 0); + pair = cfl_list_entry_first(&list->list, struct cfl_kvpair, _head); + TEST_CHECK(pair->arena == NULL); + TEST_CHECK(cfl_sds_alloc(pair->key) == 6); + cfl_kvlist_destroy(list); +} + +#define TEST_MUTATION_KEYS 48 + +static void randomized_kvlist_mutation(void) +{ + struct cfl_arena *arena; + struct cfl_kvlist *list; + struct cfl_variant *value; + char *payload; + char keys[TEST_MUTATION_KEYS][224]; + char names[TEST_MUTATION_KEYS][224]; + size_t lengths[TEST_MUTATION_KEYS]; + int present[TEST_MUTATION_KEYS]; + uint32_t state; + size_t key_length; + size_t index; + size_t slot; + size_t count; + int operation; + int round; + int expected; + + payload = malloc(4096); + TEST_CHECK(payload != NULL); + if (payload == NULL) { + return; + } + for (index = 0; index < 4096; index++) { + payload[index] = (char) ('a' + (index % 23)); + } + + /* small chunks and a low external threshold stress every storage path */ + arena = cfl_arena_create_ex(1024, 2048); + TEST_CHECK(arena != NULL); + if (arena == NULL) { + free(payload); + return; + } + + state = 2026; + for (round = 0; round < 4; round++) { + list = cfl_kvlist_create_in(arena); + TEST_CHECK(list != NULL); + if (list == NULL) { + break; + } + + for (slot = 0; slot < TEST_MUTATION_KEYS; slot++) { + key_length = 1 + test_random(&state) % 200; + memset(keys[slot], 'k', key_length); + snprintf(keys[slot], 8, "%02zu", slot); + keys[slot][2] = 'k'; + keys[slot][key_length] = '\0'; + strcpy(names[slot], keys[slot]); + present[slot] = 0; + lengths[slot] = 0; + } + + for (index = 0; index < 4000; index++) { + slot = test_random(&state) % TEST_MUTATION_KEYS; + operation = (int) (test_random(&state) % 4); + + if (operation == 0 && !present[slot]) { + lengths[slot] = test_random(&state) % 3000; + TEST_CHECK(cfl_kvlist_insert_string_s(list, names[slot], + strlen(names[slot]), + payload, lengths[slot], + CFL_FALSE) == 0); + present[slot] = 1; + } + else if (operation == 1 && present[slot]) { + TEST_CHECK(cfl_kvlist_remove(list, names[slot]) == CFL_TRUE); + present[slot] = 0; + } + else if (operation == 2 && present[slot]) { + /* rename between the short and the long spelling */ + if (strcmp(names[slot], keys[slot]) == 0) { + snprintf(names[slot], 8, "%02zu", slot); + } + else { + strcpy(names[slot], keys[slot]); + } + TEST_CHECK(cfl_kvlist_rename_s(list, keys[slot], strlen(keys[slot]), + names[slot], strlen(names[slot])) == 0 || + strcmp(names[slot], keys[slot]) == 0); + if (strcmp(names[slot], keys[slot]) == 0) { + snprintf(names[slot], 8, "%02zu", slot); + TEST_CHECK(cfl_kvlist_rename_s(list, names[slot], + strlen(names[slot]), + keys[slot], + strlen(keys[slot])) == 0); + strcpy(names[slot], keys[slot]); + } + } + else if (operation == 3 && present[slot]) { + TEST_CHECK(cfl_kvlist_remove(list, names[slot]) == CFL_TRUE); + lengths[slot] = test_random(&state) % 3000; + TEST_CHECK(cfl_kvlist_insert_string_s(list, names[slot], + strlen(names[slot]), + payload, lengths[slot], + CFL_FALSE) == 0); + } + } + + count = 0; + for (slot = 0; slot < TEST_MUTATION_KEYS; slot++) { + value = cfl_kvlist_fetch(list, names[slot]); + expected = present[slot]; + TEST_CHECK((value != NULL) == expected); + if (value == NULL || !expected) { + continue; + } + count++; + TEST_CHECK(value->type == CFL_VARIANT_STRING); + TEST_CHECK(cfl_sds_len(value->data.as_string) == lengths[slot]); + TEST_CHECK(memcmp(value->data.as_string, payload, + lengths[slot]) == 0); + TEST_CHECK(value->data.as_string[lengths[slot]] == '\0'); + } + TEST_CHECK((size_t) cfl_kvlist_count(list) == count); + + cfl_kvlist_destroy(list); + cfl_arena_reset(arena); + TEST_CHECK(cfl_arena_bytes_used(arena) == 0); + } + + cfl_arena_destroy(arena); + free(payload); +} + static void bound_external_rounding_and_cache(void) { struct cfl_arena *arena; @@ -523,8 +1113,7 @@ static void reclaim_failed_variant_construction(void) reused = cfl_variant_create_from_int64_in(arena, 1); TEST_CHECK(reused != NULL); - TEST_CHECK(((uintptr_t) reused % - offsetof(struct test_alignment_probe, value)) == 0); + TEST_CHECK(is_aligned(reused, offsetof(struct test_variant_probe, value))); cfl_variant_destroy(reused); cfl_arena_destroy(arena); } @@ -542,6 +1131,15 @@ TEST_LIST = { {"reuse_variant_and_kvpair_slots", reuse_variant_and_kvpair_slots}, {"create_like_and_rename", create_like_and_rename}, {"reuse_sds_size_classes", reuse_sds_size_classes}, + {"sds_class_boundaries", sds_class_boundaries}, + {"sds_capacity_is_writable", sds_capacity_is_writable}, + {"sds_grow_across_classes", sds_grow_across_classes}, + {"object_alignment", object_alignment}, + {"raw_alignment_after_objects", raw_alignment_after_objects}, + {"compact_short_keys", compact_short_keys}, + {"reset_reuses_storage_without_aliasing", reset_reuses_storage_without_aliasing}, + {"heap_storage_unchanged", heap_storage_unchanged}, + {"randomized_kvlist_mutation", randomized_kvlist_mutation}, {"bound_external_rounding_and_cache", bound_external_rounding_and_cache}, {"reclaim_failed_variant_construction", reclaim_failed_variant_construction}, {NULL, NULL} diff --git a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h index fd632247c6d..4bffc827c3f 100644 --- a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h +++ b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h @@ -57,6 +57,8 @@ struct cmt_msgpack_decode_context { int aggregation_type; int metric_value_type_set; int metric_section_set; + /* Appended to keep the established field layout. */ + int allow_reset; }; int cmt_decode_msgpack_create(struct cmt **out_cmt, char *in_buf, size_t in_size, diff --git a/lib/cmetrics/include/cmetrics/cmt_map.h b/lib/cmetrics/include/cmetrics/cmt_map.h index 6a3c7e79620..2d64e9b729b 100644 --- a/lib/cmetrics/include/cmetrics/cmt_map.h +++ b/lib/cmetrics/include/cmetrics/cmt_map.h @@ -51,6 +51,8 @@ struct cmt_map { size_t indexed_metric_count; /* Most recently created metric; only changed with the metric list. */ struct cmt_metric *last_metric; + /* Owned family metadata; independent of metric names and sibling scopes. */ + struct cfl_kvlist *otlp_metadata; }; struct cmt_map *cmt_map_create(int type, struct cmt_opts *opts, diff --git a/lib/cmetrics/include/cmetrics/cmt_metric.h b/lib/cmetrics/include/cmetrics/cmt_metric.h index a45eaba5b50..8499851b4eb 100644 --- a/lib/cmetrics/include/cmetrics/cmt_metric.h +++ b/lib/cmetrics/include/cmetrics/cmt_metric.h @@ -75,6 +75,9 @@ struct cmt_metric { int hash_indexed; struct cmt_map *map; struct cfl_list _hash_head; + + /* Owned OTLP datapoint metadata; independent of string label identity. */ + struct cfl_kvlist *otlp_metadata; }; struct cmt_exp_histogram_snapshot { @@ -136,6 +139,18 @@ void cmt_metric_hist_set(struct cmt_metric *metric, uint64_t timestamp, uint64_t cmt_metric_hist_get_value(struct cmt_metric *metric, int bucket_id); +/* Convert cumulative native buckets without unsigned underflow. */ +static inline uint64_t cmt_metric_hist_get_bucket_count(struct cmt_metric *metric, int bucket_id) +{ + uint64_t current; + uint64_t previous; + + current = cmt_metric_hist_get_value(metric, bucket_id); + previous = bucket_id == 0 ? 0 : cmt_metric_hist_get_value(metric, bucket_id - 1); + return current >= previous ? current - previous : 0; +} + + double cmt_metric_hist_get_sum_value(struct cmt_metric *metric); uint64_t cmt_metric_hist_get_count_value(struct cmt_metric *metric); diff --git a/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h b/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h new file mode 100644 index 00000000000..fef8eb3c57a --- /dev/null +++ b/lib/cmetrics/include/cmetrics/cmt_otlp_metadata.h @@ -0,0 +1,35 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CMT_OTLP_METADATA_H +#define CMT_OTLP_METADATA_H + +#include +#include +#include + +struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source); +int cmt_otlp_metadata_pack(mpack_writer_t *writer, struct cmt *context); +int cmt_otlp_metadata_pack_external(mpack_writer_t *writer, struct cmt *context); +int cmt_otlp_metadata_restore(struct cmt *context); +int cmt_otlp_histogram_restore(struct cmt *context); +int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample); +int cmt_otlp_attributes_match_labels(struct cmt_map *map, struct cmt_metric *sample); + +#endif diff --git a/lib/cmetrics/include/cmetrics/cmt_variant_utils.h b/lib/cmetrics/include/cmetrics/cmt_variant_utils.h index 922f870fcea..8b250419a7d 100644 --- a/lib/cmetrics/include/cmetrics/cmt_variant_utils.h +++ b/lib/cmetrics/include/cmetrics/cmt_variant_utils.h @@ -477,6 +477,7 @@ static inline int unpack_cfl_variant_string(mpack_reader_t *reader, } (*value)->type = CFL_VARIANT_STRING; + (*value)->size = value_length; return 0; } @@ -528,6 +529,7 @@ static inline int unpack_cfl_variant_binary(mpack_reader_t *reader, } (*value)->type = CFL_VARIANT_BYTES; + (*value)->size = value_length; return 0; } diff --git a/lib/cmetrics/src/CMakeLists.txt b/lib/cmetrics/src/CMakeLists.txt index a4bed296713..f73ed74dcb2 100644 --- a/lib/cmetrics/src/CMakeLists.txt +++ b/lib/cmetrics/src/CMakeLists.txt @@ -15,6 +15,7 @@ set(src cmt_summary.c cmt_histogram.c cmt_exp_histogram.c + cmt_otlp_metadata.c cmt_metric.c cmt_metric_histogram.c cmt_map.c diff --git a/lib/cmetrics/src/cmt_cat.c b/lib/cmetrics/src/cmt_cat.c index 2ab52f01274..40e090127b8 100644 --- a/lib/cmetrics/src/cmt_cat.c +++ b/lib/cmetrics/src/cmt_cat.c @@ -17,6 +17,7 @@ * limitations under the License. */ +#include #include #include #include @@ -500,6 +501,151 @@ static inline void cat_scalar_value(struct cmt_metric *metric_dst, } } +static int copy_point_metadata(struct cmt_metric *dst, struct cmt_metric *src) +{ + struct cfl_kvlist *copy; + + if (src->otlp_metadata == NULL) { + return 0; + } + + copy = cmt_otlp_metadata_clone(src->otlp_metadata); + if (copy == NULL) { + return -1; + } + + cfl_kvlist_destroy(dst->otlp_metadata); + dst->otlp_metadata = copy; + + return 0; +} + +static int sample_labels_match(struct cmt_metric *metric, int count, char **labels) +{ + int index; + struct cfl_list *head; + struct cmt_map_label *label; + + index = 0; + cfl_list_foreach(head, &metric->labels) { + if (index >= count) { + return CMT_FALSE; + } + + label = cfl_list_entry(head, struct cmt_map_label, _head); + if ((label->name == NULL) != (labels[index] == NULL)) { + return CMT_FALSE; + } + if (label->name != NULL && strcmp(label->name, labels[index]) != 0) { + return CMT_FALSE; + } + index++; + } + + return index == count; +} + +/* OTLP permits several data points of one series in a single request when + * their time windows differ; those are distinct points, not duplicates. */ +static int same_data_point(struct cmt_metric *a, struct cmt_metric *b) +{ + if (cmt_metric_get_timestamp(a) != cmt_metric_get_timestamp(b)) { + return CMT_FALSE; + } + + if (cmt_metric_has_start_timestamp(a) != cmt_metric_has_start_timestamp(b)) { + return CMT_FALSE; + } + + if (cmt_metric_has_start_timestamp(a) && + cmt_metric_get_start_timestamp(a) != cmt_metric_get_start_timestamp(b)) { + return CMT_FALSE; + } + + return CMT_TRUE; +} + +static struct cmt_metric *append_point_sample(struct cmt_map *dst, + struct cmt_metric *src, + int count, char **labels) +{ + int index; + struct cmt_metric *metric; + struct cmt_map_label *label; + + metric = calloc(1, sizeof(struct cmt_metric)); + if (metric == NULL) { + cmt_errno(); + return NULL; + } + + cfl_list_init(&metric->labels); + cfl_list_init(&metric->_hash_head); + metric->map = dst; + metric->hash = src->hash; + cfl_list_add(&metric->_head, &dst->metrics); + + for (index = 0; index < count; index++) { + label = calloc(1, sizeof(struct cmt_map_label)); + if (label == NULL) { + cmt_errno(); + cmt_map_metric_destroy(metric); + return NULL; + } + + if (labels[index] != NULL) { + label->name = cfl_sds_create(labels[index]); + if (label->name == NULL) { + free(label); + cmt_map_metric_destroy(metric); + return NULL; + } + } + cfl_list_add(&label->_head, &metric->labels); + } + + return metric; +} + +/* + * Resolve the destination for a sample carrying OTLP metadata. Like any other + * sample it merges into an existing sample with the same labels, so contexts + * concatenated more than once never duplicate a series. A new sample is only + * appended for a distinct data point (different time window) of a series. + */ +static struct cmt_metric *get_point_sample(struct cmt_opts *opts, struct cmt_map *dst, + struct cmt_metric *src, + int count, char **labels) +{ + struct cfl_list *head; + struct cmt_metric *metric; + + if (count == 0) { + /* Label-less points always merge into the static metric. */ + return cmt_map_metric_get(opts, dst, 0, NULL, CMT_TRUE); + } + + metric = cmt_map_metric_get(opts, dst, count, labels, CMT_FALSE); + if (metric == NULL) { + /* No sample has these labels yet: create an indexed one. */ + return cmt_map_metric_get(opts, dst, count, labels, CMT_TRUE); + } + + if (same_data_point(metric, src)) { + return metric; + } + + /* Rare path: a series with several data points. */ + cfl_list_foreach(head, &dst->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (sample_labels_match(metric, count, labels) && same_data_point(metric, src)) { + return metric; + } + } + + return append_point_sample(dst, src, count, labels); +} + int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map *src) { int c; @@ -508,10 +654,20 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map struct cfl_list *head; struct cmt_metric *metric_dst; struct cmt_metric *metric_src; + struct cfl_kvlist *metadata; struct cmt_summary *summary; struct cmt_histogram *histogram_src; struct cmt_histogram *histogram_dst; + if (src->otlp_metadata) { + metadata = cmt_otlp_metadata_clone(src->otlp_metadata); + if (!metadata) { + return -1; + } + cfl_kvlist_destroy(dst->otlp_metadata); + dst->otlp_metadata = metadata; + } + /* Handle static metric (no labels case) */ if (src->metric_static_set) { dst->metric_static_set = CMT_TRUE; @@ -542,6 +698,9 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } } + if (copy_point_metadata(metric_dst, metric_src) != 0) { + return -1; + } cat_scalar_value(metric_dst, metric_src); } @@ -555,7 +714,12 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } c = cfl_list_size(&metric_src->labels); - metric_dst = cmt_map_metric_get(opts, dst, c, labels, CMT_TRUE); + if (metric_src->otlp_metadata != NULL) { + metric_dst = get_point_sample(opts, dst, metric_src, c, labels); + } + else { + metric_dst = cmt_map_metric_get(opts, dst, c, labels, CMT_TRUE); + } free(labels); if (!metric_dst) { @@ -584,6 +748,9 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } } + if (copy_point_metadata(metric_dst, metric_src) != 0) { + return -1; + } cat_scalar_value(metric_dst, metric_src); } diff --git a/lib/cmetrics/src/cmt_decode_msgpack.c b/lib/cmetrics/src/cmt_decode_msgpack.c index 59f12da6218..007e847a6ad 100644 --- a/lib/cmetrics/src/cmt_decode_msgpack.c +++ b/lib/cmetrics/src/cmt_decode_msgpack.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -1220,6 +1221,16 @@ static int unpack_metric_hash(mpack_reader_t *reader, size_t index, void *contex return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->hash); } +static int unpack_metric_otlp(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context = context; + + if (decode_context->metric->otlp_metadata != NULL) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + return unpack_cfl_kvlist(reader, &decode_context->metric->otlp_metadata); +} + static int unpack_metric(mpack_reader_t *reader, struct cmt_msgpack_decode_context *decode_context, struct cmt_metric **out_metric) @@ -1241,6 +1252,7 @@ static int unpack_metric(mpack_reader_t *reader, {"histogram", unpack_metric_histogram}, {"exp_histogram", unpack_metric_exp_histogram}, {"hash", unpack_metric_hash}, + {"otlp", unpack_metric_otlp}, {NULL, NULL} }; @@ -1328,6 +1340,8 @@ static int unpack_metric(mpack_reader_t *reader, free(metric->exp_hist_negative_buckets); } + cfl_kvlist_destroy(metric->otlp_metadata); + free(metric); } else { @@ -1362,8 +1376,8 @@ static int unpack_metric_array_entry(mpack_reader_t *reader, size_t index, void old_negative_buckets = NULL; if (0 == cfl_list_size(&metric->labels)) { - /* Should we care about finding more than one "implicitly static metric" in - * the array? + /* Every label-less entry is the static metric; when several are + * present the last one wins, exactly as released readers do. */ decode_context->map->metric_static_set = 1; @@ -1440,6 +1454,9 @@ static int unpack_metric_array_entry(mpack_reader_t *reader, size_t index, void cmt_atomic_load(&metric->val_int64)); cmt_atomic_store(&decode_context->map->metric.val_uint64, cmt_atomic_load(&metric->val_uint64)); + cfl_kvlist_destroy(decode_context->map->metric.otlp_metadata); + decode_context->map->metric.otlp_metadata = metric->otlp_metadata; + metric->otlp_metadata = NULL; decode_context->map->metric.hash = metric->hash; cmt_atomic_store(&decode_context->map->metric.timestamp, cmt_atomic_load(&metric->timestamp)); @@ -1545,6 +1562,24 @@ static int unpack_meta_aggregation_type(mpack_reader_t *reader, size_t index, vo return result; } +static int unpack_meta_allow_reset(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + uint64_t value; + int result; + + decode_context = context; + result = cmt_mpack_consume_uint_tag(reader, &value); + if (result != CMT_DECODE_MSGPACK_SUCCESS) { + return result; + } + if (value > 1) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + decode_context->allow_reset = value; + return CMT_DECODE_MSGPACK_SUCCESS; +} + static int unpack_meta_opts(mpack_reader_t *reader, size_t index, void *context) { struct cmt_msgpack_decode_context *decode_context; @@ -1705,6 +1740,7 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co {"buckets", unpack_meta_buckets}, {"quantiles", unpack_meta_quantiles}, {"aggregation_type", unpack_meta_aggregation_type}, + {"allow_reset", unpack_meta_allow_reset}, {NULL, NULL} }; @@ -1761,6 +1797,7 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co else if(decode_context->map->type == CMT_COUNTER) { counter = (struct cmt_counter *) decode_context->map->parent; counter->aggregation_type = decode_context->aggregation_type; + counter->allow_reset = decode_context->allow_reset; } } @@ -2245,6 +2282,12 @@ int cmt_decode_msgpack_create(struct cmt **out_cmt, char *in_buf, size_t in_size mpack_reader_init_data(&reader, &in_buf[*offset], in_size); result = unpack_context(&reader, cmt); + if (result == CMT_DECODE_MSGPACK_SUCCESS && cmt_otlp_histogram_restore(cmt) != 0) { + result = CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; + } + if (result == CMT_DECODE_MSGPACK_SUCCESS && cmt_otlp_metadata_restore(cmt) != 0) { + result = CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; + } remainder = mpack_reader_remaining(&reader, NULL); diff --git a/lib/cmetrics/src/cmt_decode_opentelemetry.c b/lib/cmetrics/src/cmt_decode_opentelemetry.c index 6aafa4abbbf..2bb3e460257 100644 --- a/lib/cmetrics/src/cmt_decode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_decode_opentelemetry.c @@ -20,6 +20,7 @@ #include "cmt_protobuf.h" #include #include +#include #include #include #include @@ -35,6 +36,7 @@ #include #include #include +#include static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source); @@ -341,81 +343,63 @@ static uint64_t compute_metric_hash(struct cmt_map *map, struct cmt_metric *samp return cfl_hash_64bits_digest(&state); } -static char *map_type_to_key(int map_type) +static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct cmt_map *map) { - switch (map_type) { - case CMT_COUNTER: - return "counter"; - case CMT_GAUGE: - return "gauge"; - case CMT_UNTYPED: - return "untyped"; - case CMT_SUMMARY: - return "summary"; - case CMT_HISTOGRAM: - return "histogram"; - case CMT_EXP_HISTOGRAM: - return "exp_histogram"; - default: - return "unknown"; + if (!map->otlp_metadata) { + map->otlp_metadata = cfl_kvlist_create(); } + return map->otlp_metadata; } -static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct cmt_map *map) +static struct cfl_kvlist *get_or_create_data_point_metadata_context(struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample, + uint64_t timestamp) { - struct cfl_kvlist *otlp_root; - struct cfl_kvlist *metrics_root; - struct cfl_kvlist *type_root; - - if (cmt == NULL || map == NULL || map->opts == NULL || map->opts->fqname == NULL) { + if (sample == NULL) { return NULL; } - - otlp_root = get_or_create_external_metadata_kvlist(cmt->external_metadata, "otlp"); - if (otlp_root == NULL) { - return NULL; - } - - metrics_root = get_or_create_external_metadata_kvlist(otlp_root, "metrics"); - if (metrics_root == NULL) { - return NULL; + if (sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { + sample->hash = compute_metric_hash(map, sample); } - - type_root = get_or_create_external_metadata_kvlist(metrics_root, map_type_to_key(map->type)); - if (type_root == NULL) { - return NULL; + if (sample->otlp_metadata == NULL) { + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata && cmt_otlp_capture_labels(map, sample) != 0) { + return NULL; + } } - - return get_or_create_external_metadata_kvlist(type_root, map->opts->fqname); + return sample->otlp_metadata; } -static struct cfl_kvlist *get_or_create_data_point_metadata_context(struct cmt *cmt, - struct cmt_map *map, - struct cmt_metric *sample, - uint64_t timestamp) +/* Keep telemetry attributes typed; string labels remain available to native exporters. */ +static int clone_point_attributes(struct cfl_kvlist *metadata, + Opentelemetry__Proto__Common__V1__KeyValue **attributes, + size_t count) { - char key[128]; - struct cfl_kvlist *metric_context; - struct cfl_kvlist *datapoints_context; + struct cfl_kvlist *copy; + size_t index; - if (sample != NULL && sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { - sample->hash = compute_metric_hash(map, sample); + copy = cfl_kvlist_create(); + if (copy == NULL) { + return -1; } - - metric_context = get_or_create_metric_metadata_context(cmt, map); - if (metric_context == NULL) { - return NULL; + for (index = 0; index < count; index++) { + if (attributes[index] != NULL && + (attributes[index]->value == NULL || + attributes[index]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET)) { + continue; + } + if (attributes[index] == NULL || clone_kvlist_entry(copy, attributes[index]) != 0) { + cfl_kvlist_destroy(copy); + return -1; + } } - - datapoints_context = get_or_create_external_metadata_kvlist(metric_context, "datapoints"); - if (datapoints_context == NULL) { - return NULL; + if (cfl_kvlist_insert_kvlist(metadata, "attributes", copy) != 0) { + cfl_kvlist_destroy(copy); + return -1; } - - snprintf(key, sizeof(key) - 1, "%" PRIx64 ":%" PRIu64, - sample != NULL ? sample->hash : 0, timestamp); - - return get_or_create_external_metadata_kvlist(datapoints_context, key); + return 0; } static int clone_exemplars_to_kvlist(struct cfl_kvlist *target, @@ -724,6 +708,30 @@ static int decode_data_point_labels(struct cmt *cmt, return result; } +/* A repeated label-less point replaces the static sample's previous point. */ +static void reset_static_data_point(struct cmt_map *map) +{ + struct cmt_metric *metric; + struct cmt_histogram *histogram; + + metric = &map->metric; + + if (metric->otlp_metadata != NULL) { + cfl_kvlist_destroy(metric->otlp_metadata); + metric->otlp_metadata = NULL; + } + + if (map->type == CMT_HISTOGRAM && metric->hist_buckets != NULL) { + histogram = (struct cmt_histogram *) map->parent; + memset(metric->hist_buckets, 0, + sizeof(uint64_t) * (histogram->buckets->count + 1)); + } + else if (map->type == CMT_SUMMARY && metric->sum_quantiles != NULL) { + memset(metric->sum_quantiles, 0, + sizeof(uint64_t) * metric->sum_quantiles_count); + } +} + static int decode_numerical_data_point(struct cmt *cmt, struct cmt_map *map, Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point) @@ -737,9 +745,9 @@ static int decode_numerical_data_point(struct cmt *cmt, result = CMT_DECODE_OPENTELEMETRY_SUCCESS; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -768,7 +776,9 @@ static int decode_numerical_data_point(struct cmt *cmt, } else { sample = &map->metric; - + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -799,6 +809,10 @@ static int decode_numerical_data_point(struct cmt *cmt, } point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); @@ -909,9 +923,9 @@ static int decode_summary_data_point(struct cmt *cmt, static_metric_detected = CMT_FALSE; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -942,7 +956,9 @@ static int decode_summary_data_point(struct cmt *cmt, } else { sample = &map->metric; - + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -973,6 +989,10 @@ static int decode_summary_data_point(struct cmt *cmt, data_point->count); point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); @@ -1022,6 +1042,7 @@ static int decode_histogram_data_point(struct cmt *cmt, { int static_metric_detected; struct cmt_histogram *histogram; + uint64_t cumulative = 0; struct cmt_metric *sample; int result; size_t index; @@ -1079,9 +1100,9 @@ static int decode_histogram_data_point(struct cmt *cmt, static_metric_detected = CMT_FALSE; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -1112,7 +1133,9 @@ static int decode_histogram_data_point(struct cmt *cmt, } else { sample = &map->metric; - + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -1135,8 +1158,11 @@ static int decode_histogram_data_point(struct cmt *cmt, for (index = 0 ; index < data_point->n_bucket_counts; index++) { - cmt_metric_hist_set(sample, data_point->time_unix_nano, - index, data_point->bucket_counts[index]); + if (UINT64_MAX - cumulative < data_point->bucket_counts[index]) { + return CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; + } + cumulative += data_point->bucket_counts[index]; + cmt_metric_hist_set(sample, data_point->time_unix_nano, index, cumulative); } cmt_metric_hist_sum_set(sample, data_point->time_unix_nano, @@ -1145,6 +1171,10 @@ static int decode_histogram_data_point(struct cmt *cmt, data_point->count); point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); @@ -1345,9 +1375,9 @@ static int decode_exponential_histogram_data_point(struct cmt *cmt, static_metric_detected = CMT_FALSE; if (data_point->n_attributes == 0) { - if (map->metric_static_set == CMT_FALSE) { - static_metric_detected = CMT_TRUE; - } + /* Label-less points always target the static sample, the last + * one wins, as released versions behave. */ + static_metric_detected = CMT_TRUE; } if (static_metric_detected == CMT_FALSE) { @@ -1376,6 +1406,9 @@ static int decode_exponential_histogram_data_point(struct cmt *cmt, } else { sample = &map->metric; + if (map->metric_static_set) { + reset_static_data_point(map); + } map->metric_static_set = CMT_TRUE; } @@ -1438,6 +1471,10 @@ static int decode_exponential_histogram_data_point(struct cmt *cmt, struct cfl_kvlist *point_metadata; point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata == NULL || clone_point_attributes(point_metadata, data_point->attributes, + data_point->n_attributes) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } if (point_metadata != NULL) { cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); diff --git a/lib/cmetrics/src/cmt_encode_msgpack.c b/lib/cmetrics/src/cmt_encode_msgpack.c index 4341b9c651f..a9b2555f80f 100644 --- a/lib/cmetrics/src/cmt_encode_msgpack.c +++ b/lib/cmetrics/src/cmt_encode_msgpack.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -264,6 +265,7 @@ static int pack_metric(mpack_writer_t *writer, struct cmt_map *map, struct cmt_m for (index = 0 ; index <= histogram->buckets->count ; index++) { + /* Wire buckets stay cumulative, as every released reader expects. */ mpack_write_uint(writer, cmt_metric_hist_get_value(metric, index)); } @@ -477,14 +479,14 @@ static int pack_context_header(mpack_writer_t *writer, struct cmt *cmt) mpack_start_map(writer, 3); mpack_write_cstr(writer, "cmetrics"); - result = pack_cfl_variant_kvlist(writer, cmt->internal_metadata); + result = cmt_otlp_metadata_pack(writer, cmt); if (result != 0) { return -1; } mpack_write_cstr(writer, "external"); - result = pack_cfl_variant_kvlist(writer, cmt->external_metadata); + result = cmt_otlp_metadata_pack_external(writer, cmt); if (result != 0) { return -2; diff --git a/lib/cmetrics/src/cmt_encode_opentelemetry.c b/lib/cmetrics/src/cmt_encode_opentelemetry.c index 65b6399ee40..f384e13c993 100644 --- a/lib/cmetrics/src/cmt_encode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_encode_opentelemetry.c @@ -18,6 +18,7 @@ */ #include #include +#include #include #include #include @@ -82,7 +83,7 @@ struct cfl_kvlist *fetch_metadata_kvlist_key(struct cfl_kvlist *kvlist, char *ke entry_variant = cfl_kvlist_fetch(kvlist, key); - if (entry_variant != NULL) { + if (entry_variant != NULL && entry_variant->type == CFL_VARIANT_KVLIST) { entry_kvlist = entry_variant->data.as_kvlist; } else { @@ -154,6 +155,10 @@ static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, stru return NULL; } + if (map->otlp_metadata) { + return map->otlp_metadata; + } + otlp_root = fetch_metadata_kvlist_key(cmt->external_metadata, "otlp"); if (otlp_root == NULL) { return NULL; @@ -180,6 +185,10 @@ static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, struct cfl_kvlist *metric_context; struct cfl_kvlist *datapoints_context; + if (sample != NULL && sample->otlp_metadata != NULL) { + return sample->otlp_metadata; + } + metric_context = get_metric_otlp_metadata_context(cmt, map); if (metric_context == NULL) { return NULL; @@ -541,7 +550,10 @@ static size_t resolve_target_scope_index(struct cmt_opentelemetry_context *conte return 0; } - metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + metadata = fetch_metadata_kvlist_key(metric_context, "routing"); + if (metadata == NULL) { + metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + } if (metadata == NULL) { return 0; } @@ -1294,6 +1306,174 @@ static inline Opentelemetry__Proto__Common__V1__KeyValue **cfl_kvlist_to_otlp_kv } +static void *attribute_arena_alloc(void *data, size_t size) +{ + return cfl_arena_malloc(data, size); +} + +static void attribute_arena_free(void *data, void *pointer) +{ + /* The encoder releases all datapoint objects with their arena. */ +} + +/* + * TRUE when every value can be represented as an OTLP AnyValue: NULL values, + * unsigned integers above INT64_MAX and unknown types have no OTLP form. + */ +static int otlp_variant_is_representable(struct cfl_variant *value, int depth) +{ + size_t index; + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_array *array; + + if (value == NULL || depth > 64) { + return CMT_FALSE; + } + + switch (value->type) { + case CFL_VARIANT_STRING: + case CFL_VARIANT_BOOL: + case CFL_VARIANT_INT: + case CFL_VARIANT_DOUBLE: + case CFL_VARIANT_BYTES: + case CFL_VARIANT_REFERENCE: + return CMT_TRUE; + case CFL_VARIANT_UINT: + return value->data.as_uint64 <= INT64_MAX; + case CFL_VARIANT_ARRAY: + array = value->data.as_array; + for (index = 0; index < array->entry_count; index++) { + if (!otlp_variant_is_representable(array->entries[index], depth + 1)) { + return CMT_FALSE; + } + } + return CMT_TRUE; + case CFL_VARIANT_KVLIST: + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (!otlp_variant_is_representable(pair->val, depth + 1)) { + return CMT_FALSE; + } + } + return CMT_TRUE; + default: + return CMT_FALSE; + } +} + +static int otlp_kvlist_is_representable(struct cfl_kvlist *kvlist) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + + cfl_list_foreach(head, &kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (!otlp_variant_is_representable(pair->val, 0)) { + return CMT_FALSE; + } + } + + return CMT_TRUE; +} + +static int apply_typed_point_attributes(struct cmt_opentelemetry_context *context, + struct cmt_map *map, struct cmt_metric *sample, + void *data_point) +{ + struct cfl_kvlist *metadata; + struct cfl_kvlist *attributes; + struct cfl_arena *arena; + struct cfl_list *head; + struct cmt_label *label; + ProtobufCAllocator allocator; + Opentelemetry__Proto__Common__V1__KeyValue **heap; + Opentelemetry__Proto__Common__V1__KeyValue **replacement; + Opentelemetry__Proto__Common__V1__KeyValue ***target; + size_t *target_count; + uint8_t *buffer; + size_t count; + size_t size; + size_t index; + size_t static_count; + + metadata = get_data_point_otlp_metadata_context(context->cmt, map, sample); + attributes = fetch_metadata_kvlist_key(metadata, "attributes"); + if (attributes == NULL || !cmt_otlp_attributes_match_labels(map, sample)) { + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; + } + + /* typed values without an OTLP form keep the label derived attributes */ + if (!otlp_kvlist_is_representable(attributes)) { + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; + } + count = cfl_kvlist_count(attributes); + static_count = cfl_list_size(&context->cmt->static_labels->list); + heap = cfl_kvlist_to_otlp_kvpair_list(attributes); + if (heap == NULL) { + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + arena = get_context_arena(context); + allocator.alloc = attribute_arena_alloc; + allocator.free = attribute_arena_free; + allocator.allocator_data = arena; + replacement = cfl_arena_calloc(arena, count + static_count + 1, sizeof(*replacement)); + if (replacement == NULL) { + otlp_kvpair_list_destroy(heap, count); + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + for (index = 0; index < count; index++) { + size = opentelemetry__proto__common__v1__key_value__get_packed_size(heap[index]); + buffer = malloc(size ? size : 1); + if (buffer == NULL) { + otlp_kvpair_list_destroy(heap, count); + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + opentelemetry__proto__common__v1__key_value__pack(heap[index], buffer); + replacement[index] = opentelemetry__proto__common__v1__key_value__unpack( + &allocator, size, buffer); + free(buffer); + if (replacement[index] == NULL) { + otlp_kvpair_list_destroy(heap, count); + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + } + otlp_kvpair_list_destroy(heap, count); + index = count; + cfl_list_foreach(head, &context->cmt->static_labels->list) { + label = cfl_list_entry(head, struct cmt_label, _head); + replacement[index] = initialize_string_attribute(arena, label->key, label->val); + if (replacement[index++] == NULL) { + return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + } + switch (map->type) { + case CMT_COUNTER: + case CMT_GAUGE: + case CMT_UNTYPED: + target = &((Opentelemetry__Proto__Metrics__V1__NumberDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__NumberDataPoint *) data_point)->n_attributes; + break; + case CMT_HISTOGRAM: + target = &((Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *) data_point)->n_attributes; + break; + case CMT_EXP_HISTOGRAM: + target = &((Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *) data_point)->n_attributes; + break; + case CMT_SUMMARY: + target = &((Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *) data_point)->attributes; + target_count = &((Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *) data_point)->n_attributes; + break; + default: + return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; + } + *target = replacement; + *target_count = count + static_count; + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; +} + static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_kvlist_to_otlp_any_value(struct cfl_variant *value) { size_t entry_count; @@ -1453,6 +1633,7 @@ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_binary_to_ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_to_otlp_any_value(struct cfl_variant *value) { Opentelemetry__Proto__Common__V1__AnyValue *result; + struct cfl_variant signed_value; if (value->type == CFL_VARIANT_STRING) { result = cfl_variant_string_to_otlp_any_value(value); @@ -1463,6 +1644,17 @@ static inline Opentelemetry__Proto__Common__V1__AnyValue *cfl_variant_to_otlp_an else if (value->type == CFL_VARIANT_INT) { result = cfl_variant_int64_to_otlp_any_value(value); } + else if (value->type == CFL_VARIANT_UINT) { + if (value->data.as_uint64 > INT64_MAX) { + result = NULL; + } + else { + signed_value = *value; + signed_value.type = CFL_VARIANT_INT; + signed_value.data.as_int64 = (int64_t) value->data.as_uint64; + result = cfl_variant_int64_to_otlp_any_value(&signed_value); + } + } else if (value->type == CFL_VARIANT_DOUBLE) { result = cfl_variant_double_to_otlp_any_value(value); } @@ -1518,6 +1710,10 @@ static int64_t fetch_metadata_int64_key(struct cfl_kvlist *metadata, char *key_n return 0; } + if (value->type == CFL_VARIANT_UINT && value->data.as_uint64 <= INT64_MAX) { + return (int64_t) value->data.as_uint64; + } + if (value->type != CFL_VARIANT_INT) { *error_flag = CMT_TRUE; @@ -1751,11 +1947,11 @@ static Opentelemetry__Proto__Metrics__V1__ResourceMetrics ** void destroy_instrumentation_scope(Opentelemetry__Proto__Common__V1__InstrumentationScope *scope) { - if (scope->name != NULL) { + if (is_string_releaseable(scope->name)) { cfl_sds_destroy(scope->name); } - if (scope->version != NULL) { + if (is_string_releaseable(scope->version)) { cfl_sds_destroy(scope->version); } @@ -2367,7 +2563,9 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint * if (bucket_list != NULL) { for (index = 0 ; index < bucket_count ; index++) { - data_point->bucket_counts[index] = bucket_list[index]; + data_point->bucket_counts[index] = index == 0 ? bucket_list[index] : + (bucket_list[index] >= bucket_list[index - 1] ? + bucket_list[index] - bucket_list[index - 1] : 0); } } } @@ -3417,6 +3615,11 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, } } + result = apply_typed_point_attributes(context, map, sample, data_point); + if (result != CMT_ENCODE_OPENTELEMETRY_SUCCESS) { + destroy_data_point(data_point, map->type); + return result; + } apply_data_point_metadata_from_otlp_context(context->cmt, map, sample, data_point); result = append_data_point_to_metric(metric, (void *) data_point, sample_index); diff --git a/lib/cmetrics/src/cmt_encode_text.c b/lib/cmetrics/src/cmt_encode_text.c index 7d499d72cee..1835c11461e 100644 --- a/lib/cmetrics/src/cmt_encode_text.c +++ b/lib/cmetrics/src/cmt_encode_text.c @@ -188,6 +188,10 @@ static struct cfl_kvlist *get_data_point_metadata_context(struct cmt *cmt, const char *type_key; char key[128]; + if (metric != NULL && metric->otlp_metadata != NULL) { + return metric->otlp_metadata; + } + type_key = map_type_to_otlp_key(map->type); if (type_key == NULL) { return NULL; diff --git a/lib/cmetrics/src/cmt_map.c b/lib/cmetrics/src/cmt_map.c index fb4fe9a150e..ccec20fc464 100644 --- a/lib/cmetrics/src/cmt_map.c +++ b/lib/cmetrics/src/cmt_map.c @@ -46,6 +46,8 @@ static void metric_release_storage(struct cmt_metric *metric) free(metric->exp_hist_positive_buckets); free(metric->exp_hist_negative_buckets); free(metric->sum_quantiles); + cfl_kvlist_destroy(metric->otlp_metadata); + metric->otlp_metadata = NULL; metric->hist_buckets = NULL; metric->exp_hist_positive_buckets = NULL; @@ -506,16 +508,13 @@ void cmt_map_destroy(struct cmt_map *map) } /* histogram and quantile allocation for static metric */ - if (map->metric_static_set) { - metric = &map->metric; - - metric_release_storage(metric); - } + metric_release_storage(&map->metric); if (map->unit != NULL) { cfl_sds_destroy(map->unit); } + cfl_kvlist_destroy(map->otlp_metadata); free(map->metric_buckets); free(map); diff --git a/lib/cmetrics/src/cmt_otlp_metadata.c b/lib/cmetrics/src/cmt_otlp_metadata.c new file mode 100644 index 00000000000..f0501a58ba4 --- /dev/null +++ b/lib/cmetrics/src/cmt_otlp_metadata.c @@ -0,0 +1,941 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#define METADATA_KEY "__cmetrics_native_metadata_v1" +/* Written only by unreleased builds; ignored on input, never emitted. */ +#define HISTOGRAM_MARKER_KEY "__cmetrics_histogram_buckets" + +static struct cfl_kvlist *clone_kvlist(struct cfl_kvlist *source); + +static struct cfl_variant *clone_variant(struct cfl_variant *source) +{ + size_t index; + struct cfl_array *array; + struct cfl_kvlist *kvlist; + struct cfl_variant *entry; + struct cfl_variant *copy; + + switch (source->type) { + case CFL_VARIANT_STRING: + return cfl_variant_create_from_string_s(source->data.as_string, + cfl_variant_size_get(source), CFL_FALSE); + case CFL_VARIANT_BYTES: + return cfl_variant_create_from_bytes(source->data.as_bytes, + cfl_variant_size_get(source), CFL_FALSE); + case CFL_VARIANT_BOOL: + return cfl_variant_create_from_bool(source->data.as_bool); + case CFL_VARIANT_INT: + return cfl_variant_create_from_int64(source->data.as_int64); + case CFL_VARIANT_UINT: + return cfl_variant_create_from_uint64(source->data.as_uint64); + case CFL_VARIANT_DOUBLE: + return cfl_variant_create_from_double(source->data.as_double); + case CFL_VARIANT_NULL: + return cfl_variant_create_from_null(); + case CFL_VARIANT_ARRAY: + array = cfl_array_create(source->data.as_array->entry_count); + if (array == NULL) { + return NULL; + } + for (index = 0; index < source->data.as_array->entry_count; index++) { + entry = clone_variant(source->data.as_array->entries[index]); + if (entry == NULL || cfl_array_append(array, entry) != 0) { + cfl_variant_destroy(entry); + cfl_array_destroy(array); + return NULL; + } + } + copy = cfl_variant_create_from_array(array); + if (copy == NULL) { + cfl_array_destroy(array); + } + return copy; + case CFL_VARIANT_KVLIST: + kvlist = clone_kvlist(source->data.as_kvlist); + if (kvlist == NULL) { + return NULL; + } + copy = cfl_variant_create_from_kvlist(kvlist); + if (copy == NULL) { + cfl_kvlist_destroy(kvlist); + } + return copy; + default: + return NULL; + } +} + +static struct cfl_kvlist *clone_kvlist(struct cfl_kvlist *source) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_kvlist *copy; + struct cfl_variant *value; + + copy = cfl_kvlist_create(); + if (copy == NULL) { + return NULL; + } + cfl_list_foreach(head, &source->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + value = clone_variant(pair->val); + if (value == NULL || + cfl_kvlist_insert_s(copy, pair->key, cfl_sds_len(pair->key), value) != 0) { + cfl_variant_destroy(value); + cfl_kvlist_destroy(copy); + return NULL; + } + } + return copy; +} + +/* + * Deep copy, preserving key order and duplicate keys. Unlike a msgpack round + * trip it has no nesting limit, so metadata accepted by a decoder can always + * be copied. + */ +struct cfl_kvlist *cmt_otlp_metadata_clone(struct cfl_kvlist *source) +{ + if (source == NULL) { + return NULL; + } + return clone_kvlist(source); +} + +static int visit_maps(struct cmt *context, + int (*callback)(struct cmt_map *, void *), void *data) +{ + struct cfl_list *head; + int result; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_summary *summary; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; + + + cfl_list_foreach(head, &context->counters) { + counter = cfl_list_entry(head, struct cmt_counter, _head); + result = callback(counter->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->gauges) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + result = callback(gauge->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->untypeds) { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + result = callback(untyped->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->summaries) { + summary = cfl_list_entry(head, struct cmt_summary, _head); + result = callback(summary->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + result = callback(histogram->map, data); + if (result != 0) { + return result; + } + } + cfl_list_foreach(head, &context->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + result = callback(exp_histogram->map, data); + if (result != 0) { + return result; + } + } + return 0; +} + +static int count_maps(struct cmt_map *map, void *data) +{ + (*(size_t *) data)++; + return 0; +} + +static int has_metadata(struct cmt_map *map, void *data) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + if (map->otlp_metadata || map->metric.otlp_metadata || + (map->type == CMT_COUNTER && ((struct cmt_counter *) map->parent)->allow_reset)) { + return 1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (sample->otlp_metadata) { + return 1; + } + } + return 0; +} + +static int pack_optional_map(mpack_writer_t *writer, struct cfl_kvlist *map) +{ + mpack_writer_t metadata_writer; + char *buffer = NULL; + size_t size = 0; + int result; + + if (map == NULL) { + mpack_write_nil(writer); + return 0; + } + /* A binary sidecar does not add nesting to the established wire metadata + * tree. Older readers can retain/ignore it even for deeply nested values. */ + mpack_writer_init_growable(&metadata_writer, &buffer, &size); + result = pack_cfl_variant_kvlist(&metadata_writer, map); + if (mpack_writer_destroy(&metadata_writer) != mpack_ok || result || size > UINT32_MAX) { + free(buffer); + return -1; + } + mpack_write_bin(writer, buffer, size); + free(buffer); + return 0; +} + +static int pack_map_metadata(struct cmt_map *map, void *data) +{ + mpack_writer_t *writer = data; + struct cfl_list *head; + struct cmt_metric *sample; + + mpack_start_map(writer, 5); + mpack_write_cstr(writer, "type"); + mpack_write_int(writer, map->type); + mpack_write_cstr(writer, "name"); + mpack_write_cstr(writer, map->opts->fqname); + mpack_write_cstr(writer, "family"); + if (pack_optional_map(writer, map->otlp_metadata) != 0) { + return -1; + } + mpack_write_cstr(writer, "allow_reset"); + mpack_write_bool(writer, map->type == CMT_COUNTER && + ((struct cmt_counter *) map->parent)->allow_reset); + mpack_write_cstr(writer, "points"); + mpack_start_array(writer, cfl_list_size(&map->metrics) + !!map->metric_static_set); + if (map->metric_static_set && pack_optional_map(writer, map->metric.otlp_metadata) != 0) { + return -1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (pack_optional_map(writer, sample->otlp_metadata) != 0) { + return -1; + } + } + mpack_finish_array(writer); + mpack_finish_map(writer); + return 0; +} + +/* The established internal metadata map accepts arbitrary typed keys. No new + * metric/map wire fields are emitted, so old decoders still consume the chunk. + * Histogram buckets are never re-shaped here: they stay cumulative on the wire, + * exactly as released readers and writers expect. */ +int cmt_otlp_metadata_pack(mpack_writer_t *writer, struct cmt *context) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + size_t count = 0; + size_t entries; + int has_sidecar; + int has_marker; + + has_sidecar = visit_maps(context, has_metadata, NULL) != 0; + has_marker = cfl_kvlist_fetch(context->internal_metadata, + HISTOGRAM_MARKER_KEY) != NULL; + if (!has_sidecar && !has_marker) { + return pack_cfl_variant_kvlist(writer, context->internal_metadata); + } + if (has_sidecar && cfl_kvlist_fetch(context->internal_metadata, METADATA_KEY)) { + return -1; + } + + /* A stale bucket marker is never forwarded. */ + entries = has_sidecar; + cfl_list_foreach(head, &context->internal_metadata->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (strcmp(pair->key, HISTOGRAM_MARKER_KEY) != 0) { + entries++; + } + } + + mpack_start_map(writer, entries); + cfl_list_foreach(head, &context->internal_metadata->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (strcmp(pair->key, HISTOGRAM_MARKER_KEY) == 0) { + continue; + } + mpack_write_str(writer, pair->key, cfl_sds_len(pair->key)); + if (pack_cfl_variant(writer, pair->val) != 0) { + return -1; + } + } + + if (!has_sidecar) { + mpack_finish_map(writer); + return 0; + } + + mpack_write_cstr(writer, METADATA_KEY); + visit_maps(context, count_maps, &count); + mpack_start_array(writer, count); + if (visit_maps(context, pack_map_metadata, writer) != 0) { + return -1; + } + mpack_finish_array(writer); + mpack_finish_map(writer); + return 0; +} + +/* + * Fluent Bit 5.1 and older keep OTLP point metadata in the external metadata + * tree: otlp -> metrics -> -> -> {metadata, datapoints}, where a + * point is keyed by ":". They do not read the binary + * sidecar, so the encoder also writes that tree for families with metadata, + * with the keys those readers use. The tree is built on a copy: the context + * itself is not modified. + */ +static const char *legacy_point_keys[] = { + "start_time_unix_nano", "flags", "number_value_case", "has_sum", "has_min", + "min", "has_max", "max", "exemplars", NULL +}; + +static const char *legacy_type_key(int map_type) +{ + switch (map_type) { + case CMT_COUNTER: + return "counter"; + case CMT_GAUGE: + return "gauge"; + case CMT_UNTYPED: + return "untyped"; + case CMT_SUMMARY: + return "summary"; + case CMT_HISTOGRAM: + return "histogram"; + case CMT_EXP_HISTOGRAM: + return "exp_histogram"; + default: + return "unknown"; + } +} + +static struct cfl_kvlist *legacy_child(struct cfl_kvlist *parent, const char *key) +{ + struct cfl_variant *value; + struct cfl_kvlist *child; + + value = cfl_kvlist_fetch(parent, (char *) key); + if (value != NULL && value->type == CFL_VARIANT_KVLIST) { + return value->data.as_kvlist; + } + if (value != NULL) { + cfl_kvlist_remove(parent, (char *) key); + } + child = cfl_kvlist_create(); + if (child == NULL) { + return NULL; + } + if (cfl_kvlist_insert_kvlist(parent, (char *) key, child) != 0) { + cfl_kvlist_destroy(child); + return NULL; + } + return child; +} + +static int legacy_add_point(struct cfl_kvlist *datapoints, struct cmt_metric *sample) +{ + char key[64]; + size_t index; + struct cfl_variant *value; + struct cfl_variant *copy; + struct cfl_kvlist *point; + + if (sample->otlp_metadata == NULL) { + return 0; + } + + snprintf(key, sizeof(key), "%" PRIx64 ":%" PRIu64, + sample->hash, cmt_metric_get_timestamp(sample)); + cfl_kvlist_remove(datapoints, key); + point = legacy_child(datapoints, key); + if (point == NULL) { + return -1; + } + + for (index = 0; legacy_point_keys[index] != NULL; index++) { + value = cfl_kvlist_fetch(sample->otlp_metadata, (char *) legacy_point_keys[index]); + if (value == NULL) { + continue; + } + copy = clone_variant(value); + if (copy == NULL || + cfl_kvlist_insert(point, (char *) legacy_point_keys[index], copy) != 0) { + cfl_variant_destroy(copy); + return -1; + } + } + return 0; +} + +static int legacy_add_map(struct cmt_map *map, void *data) +{ + struct cfl_kvlist *metrics = data; + struct cfl_kvlist *type_root; + struct cfl_kvlist *family; + struct cfl_kvlist *datapoints; + struct cfl_kvlist *copy; + struct cfl_variant *metadata; + struct cfl_list *head; + struct cmt_metric *sample; + int points; + + metadata = map->otlp_metadata ? cfl_kvlist_fetch(map->otlp_metadata, "metadata") : NULL; + points = map->metric_static_set && map->metric.otlp_metadata != NULL; + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + points |= sample->otlp_metadata != NULL; + } + if (!points && (metadata == NULL || metadata->type != CFL_VARIANT_KVLIST)) { + return 0; + } + + type_root = legacy_child(metrics, legacy_type_key(map->type)); + if (type_root == NULL) { + return -1; + } + family = legacy_child(type_root, map->opts->fqname); + if (family == NULL) { + return -1; + } + + if (metadata != NULL && metadata->type == CFL_VARIANT_KVLIST) { + copy = clone_kvlist(metadata->data.as_kvlist); + cfl_kvlist_remove(family, "metadata"); + if (copy == NULL || cfl_kvlist_insert_kvlist(family, "metadata", copy) != 0) { + cfl_kvlist_destroy(copy); + return -1; + } + } + + if (!points) { + return 0; + } + datapoints = legacy_child(family, "datapoints"); + if (datapoints == NULL) { + return -1; + } + if (map->metric_static_set && legacy_add_point(datapoints, &map->metric) != 0) { + return -1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (legacy_add_point(datapoints, sample) != 0) { + return -1; + } + } + return 0; +} + +static int has_legacy_metadata(struct cmt_map *map, void *data) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + if ((map->otlp_metadata && cfl_kvlist_fetch(map->otlp_metadata, "metadata")) || + (map->metric_static_set && map->metric.otlp_metadata)) { + return 1; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (sample->otlp_metadata) { + return 1; + } + } + return 0; +} + +int cmt_otlp_metadata_pack_external(mpack_writer_t *writer, struct cmt *context) +{ + int result; + struct cfl_kvlist *external; + struct cfl_kvlist *otlp; + struct cfl_kvlist *metrics; + + if (visit_maps(context, has_legacy_metadata, NULL) == 0) { + return pack_cfl_variant_kvlist(writer, context->external_metadata); + } + + external = clone_kvlist(context->external_metadata); + if (external == NULL) { + return -1; + } + otlp = legacy_child(external, "otlp"); + metrics = otlp ? legacy_child(otlp, "metrics") : NULL; + if (metrics == NULL || visit_maps(context, legacy_add_map, metrics) != 0) { + cfl_kvlist_destroy(external); + return -1; + } + + result = pack_cfl_variant_kvlist(writer, external); + cfl_kvlist_destroy(external); + return result; +} + +struct staged_family_metadata { + struct cmt_map *map; + struct cfl_kvlist *family; + struct cfl_kvlist **points; + size_t point_count; + int allow_reset; + int restore_points; +}; + +struct restore_state { + struct cfl_array *families; + size_t index; + struct staged_family_metadata *staged; +}; + +static int restore_optional_map(struct cfl_kvlist **target, struct cfl_variant *value) +{ + struct cfl_kvlist *copy = NULL; + mpack_reader_t reader; + size_t remaining; + int result; + + if (value == NULL || value->type == CFL_VARIANT_NULL) { + return 0; + } + if (value->type == CFL_VARIANT_BYTES) { + mpack_reader_init_data(&reader, value->data.as_bytes, cfl_sds_len(value->data.as_bytes)); + result = unpack_cfl_kvlist(&reader, ©); + remaining = result == 0 ? mpack_reader_remaining(&reader, NULL) : 0; + if (mpack_reader_destroy(&reader) != mpack_ok || result || remaining) { + cfl_kvlist_destroy(copy); + return -1; + } + } + else { + return -1; + } + if (!copy) { + return -1; + } + cfl_kvlist_destroy(*target); + *target = copy; + return 0; +} + +static int stage_map_metadata(struct cmt_map *map, void *data) +{ + struct restore_state *state = data; + struct staged_family_metadata *staged; + struct cfl_variant *family; + struct cfl_variant *points; + struct cfl_variant *reset; + size_t index; + + staged = &state->staged[state->index]; + family = state->families->entries[state->index++]; + staged->map = map; + points = cfl_kvlist_fetch(family->data.as_kvlist, "points"); + reset = cfl_kvlist_fetch(family->data.as_kvlist, "allow_reset"); + if (!reset || reset->type != CFL_VARIANT_BOOL) { + return -1; + } + staged->allow_reset = reset->data.as_bool; + + /* metadata that cannot be read back is skipped for this family only */ + if (restore_optional_map(&staged->family, + cfl_kvlist_fetch(family->data.as_kvlist, "family")) != 0) { + staged->family = NULL; + } + + /* Label-less entries fold into the static metric on decode, so the point + * list of that family may no longer line up: keep the family metadata + * but skip its point metadata rather than misattributing it. */ + staged->restore_points = points->data.as_array->entry_count == + cfl_list_size(&map->metrics) + !!map->metric_static_set; + if (!staged->restore_points) { + return 0; + } + + staged->point_count = points->data.as_array->entry_count; + if (staged->point_count > 0) { + staged->points = calloc(staged->point_count, sizeof(*staged->points)); + if (!staged->points) { + return -1; + } + } + for (index = 0; index < staged->point_count; index++) { + /* a point whose metadata cannot be read back keeps none */ + if (restore_optional_map(&staged->points[index], + points->data.as_array->entries[index]) != 0) { + staged->points[index] = NULL; + } + } + return 0; +} + +static void commit_map_metadata(struct staged_family_metadata *staged) +{ + struct cmt_map *map = staged->map; + struct cfl_list *head; + struct cmt_metric *sample; + size_t index = 0; + + cfl_kvlist_destroy(map->otlp_metadata); + map->otlp_metadata = staged->family; + staged->family = NULL; + if (map->type == CMT_COUNTER) { + ((struct cmt_counter *) map->parent)->allow_reset = staged->allow_reset; + } + if (!staged->restore_points) { + return; + } + if (map->metric_static_set) { + cfl_kvlist_destroy(map->metric.otlp_metadata); + map->metric.otlp_metadata = staged->points[index]; + staged->points[index++] = NULL; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = staged->points[index]; + staged->points[index++] = NULL; + } +} + +static int check_map_metadata_shape(struct cmt_map *map, void *data) +{ + struct restore_state *state = data; + struct cfl_variant *family; + struct cfl_variant *points; + struct cfl_variant *name; + struct cfl_variant *type; + + if (state->index >= state->families->entry_count) { + return -1; + } + family = state->families->entries[state->index++]; + if (family->type != CFL_VARIANT_KVLIST) { + return -1; + } + points = cfl_kvlist_fetch(family->data.as_kvlist, "points"); + name = cfl_kvlist_fetch(family->data.as_kvlist, "name"); + type = cfl_kvlist_fetch(family->data.as_kvlist, "type"); + if (points == NULL || points->type != CFL_VARIANT_ARRAY) { + return -1; + } + if (name != NULL && (name->type != CFL_VARIANT_STRING || + cfl_sds_len(name->data.as_string) != cfl_sds_len(map->opts->fqname) || + memcmp(name->data.as_string, map->opts->fqname, + cfl_sds_len(map->opts->fqname)) != 0)) { + return -1; + } + if (type != NULL && !((type->type == CFL_VARIANT_INT && type->data.as_int64 == map->type) || + (type->type == CFL_VARIANT_UINT && type->data.as_uint64 == map->type))) { + return -1; + } + return 0; +} + +static void drop_legacy_family(struct cmt *context, struct cmt_map *map) +{ + struct cfl_variant *otlp; + struct cfl_variant *metrics; + struct cfl_variant *type_root; + + otlp = cfl_kvlist_fetch(context->external_metadata, "otlp"); + if (otlp == NULL || otlp->type != CFL_VARIANT_KVLIST) { + return; + } + metrics = cfl_kvlist_fetch(otlp->data.as_kvlist, "metrics"); + if (metrics == NULL || metrics->type != CFL_VARIANT_KVLIST) { + return; + } + type_root = cfl_kvlist_fetch(metrics->data.as_kvlist, (char *) legacy_type_key(map->type)); + if (type_root == NULL || type_root->type != CFL_VARIANT_KVLIST) { + return; + } + cfl_kvlist_remove(type_root->data.as_kvlist, map->opts->fqname); +} + +int cmt_otlp_metadata_restore(struct cmt *context) +{ + struct cfl_variant *value; + struct restore_state state; + int result; + size_t family_index; + size_t point_index; + + value = cfl_kvlist_fetch(context->internal_metadata, METADATA_KEY); + if (value == NULL) { + return 0; + } + if (value->type != CFL_VARIANT_ARRAY) { + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + return 0; + } + state.families = value->data.as_array; + state.index = 0; + result = visit_maps(context, check_map_metadata_shape, &state); + if (result != 0 || state.index != state.families->entry_count) { + /* Optional metadata must not invalidate an otherwise usable chunk. */ + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + return 0; + } + state.staged = calloc(state.families->entry_count, sizeof(*state.staged)); + if (!state.staged && state.families->entry_count > 0) { + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + return 0; + } + state.index = 0; + result = visit_maps(context, stage_map_metadata, &state); + if (result == 0) { + for (family_index = 0; family_index < state.families->entry_count; family_index++) { + /* the legacy copy written for older readers is superseded */ + drop_legacy_family(context, state.staged[family_index].map); + commit_map_metadata(&state.staged[family_index]); + } + } + for (family_index = 0; family_index < state.families->entry_count; family_index++) { + cfl_kvlist_destroy(state.staged[family_index].family); + if (state.staged[family_index].points != NULL) { + for (point_index = 0; point_index < state.staged[family_index].point_count; point_index++) { + cfl_kvlist_destroy(state.staged[family_index].points[point_index]); + } + } + free(state.staged[family_index].points); + } + free(state.staged); + cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); + /* Malformed optional metadata never invalidates the native metric payload. */ + return 0; +} + +int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample) +{ + struct cfl_list *key_head; + struct cfl_list *value_head; + struct cmt_map_label *key; + struct cmt_map_label *value; + struct cfl_kvlist *projection; + + if (!sample->otlp_metadata) { + return -1; + } + projection = cfl_kvlist_create(); + if (!projection) { + return -1; + } + key_head = map->label_keys.next; + value_head = sample->labels.next; + while (key_head != &map->label_keys && value_head != &sample->labels) { + key = cfl_list_entry(key_head, struct cmt_map_label, _head); + value = cfl_list_entry(value_head, struct cmt_map_label, _head); + if (key->name && value->name && cfl_kvlist_insert_string(projection, key->name, value->name)) { + cfl_kvlist_destroy(projection); + return -1; + } + key_head = key_head->next; + value_head = value_head->next; + } + cfl_kvlist_remove(sample->otlp_metadata, "label_projection"); + if (cfl_kvlist_insert_kvlist(sample->otlp_metadata, "label_projection", projection)) { + cfl_kvlist_destroy(projection); + return -1; + } + return 0; +} + +int cmt_otlp_attributes_match_labels(struct cmt_map *map, struct cmt_metric *sample) +{ + struct cfl_variant *projection; + struct cfl_list *key_head; + struct cfl_list *value_head; + struct cfl_list *original_head; + struct cfl_kvpair *original; + struct cmt_map_label *key; + struct cmt_map_label *value; + + if (!sample->otlp_metadata) { + return CMT_FALSE; + } + projection = cfl_kvlist_fetch(sample->otlp_metadata, "label_projection"); + if (!projection || projection->type != CFL_VARIANT_KVLIST) { + return CMT_FALSE; + } + key_head = map->label_keys.next; + value_head = sample->labels.next; + original_head = projection->data.as_kvlist->list.next; + while (key_head != &map->label_keys && value_head != &sample->labels) { + key = cfl_list_entry(key_head, struct cmt_map_label, _head); + value = cfl_list_entry(value_head, struct cmt_map_label, _head); + if (key->name && value->name) { + if (original_head == &projection->data.as_kvlist->list) { + return CMT_FALSE; + } + original = cfl_list_entry(original_head, struct cfl_kvpair, _head); + if (original->val->type != CFL_VARIANT_STRING || + strcmp(original->key, key->name) || strcmp(original->val->data.as_string, value->name)) { + return CMT_FALSE; + } + original_head = original_head->next; + } + key_head = key_head->next; + value_head = value_head->next; + } + return original_head == &projection->data.as_kvlist->list && value_head == &sample->labels; +} + +/* + * CMetrics stores classic histogram buckets cumulatively (including +Inf) and + * the msgpack wire format carries them exactly like that. Fluent Bit v5.1.0 + * and older, however, stored OTLP-decoded histograms with independent bucket + * counts in the same fields, so chunks written by those releases (forward + * peers, filesystem backlog) can contain either shape without any marker. + * + * Detect the shape from the sample itself instead of trusting a marker: + * + * - monotonic buckets whose last (+Inf) value equals count: cumulative. + * - otherwise, buckets whose sum equals count: independent counts. + * - anything else is left untouched (cumulative by default). + * + * Both rules can only match at once when every bucket but +Inf is zero, in + * which case both interpretations are identical, so the check is unambiguous + * for consistent data. + */ +static int histogram_sample_is_independent(struct cmt_metric *sample, size_t count) +{ + uint64_t total; + uint64_t value; + uint64_t previous; + uint64_t sample_count; + size_t index; + int monotonic; + + if (sample->hist_buckets == NULL || count == 0) { + return CMT_FALSE; + } + + sample_count = cmt_metric_hist_get_count_value(sample); + total = 0; + previous = 0; + monotonic = CMT_TRUE; + + for (index = 0; index < count; index++) { + value = sample->hist_buckets[index]; + if (value < previous) { + monotonic = CMT_FALSE; + } + previous = value; + + if (value > UINT64_MAX - total) { + /* An overflowing sum cannot match any count. */ + return CMT_FALSE; + } + total += value; + } + + if (monotonic && sample->hist_buckets[count - 1] == sample_count) { + return CMT_FALSE; + } + + return total == sample_count; +} + +static void normalize_histogram_sample(struct cmt_metric *sample, size_t count) +{ + uint64_t total; + size_t index; + + if (!histogram_sample_is_independent(sample, count)) { + return; + } + + total = 0; + for (index = 0; index < count; index++) { + /* No overflow: the sum was validated against count. */ + total += sample->hist_buckets[index]; + sample->hist_buckets[index] = total; + } +} + +int cmt_otlp_histogram_restore(struct cmt *context) +{ + struct cfl_list *head; + struct cfl_list *sample_head; + struct cmt_histogram *histogram; + struct cmt_metric *sample; + + /* A bucket marker is advisory at most: drop it whatever its value. */ + if (cfl_kvlist_fetch(context->internal_metadata, HISTOGRAM_MARKER_KEY) != NULL) { + cfl_kvlist_remove(context->internal_metadata, HISTOGRAM_MARKER_KEY); + } + + cfl_list_foreach(head, &context->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + if (histogram->buckets == NULL) { + continue; + } + + if (histogram->map->metric_static_set) { + normalize_histogram_sample(&histogram->map->metric, + histogram->buckets->count + 1); + } + + cfl_list_foreach(sample_head, &histogram->map->metrics) { + sample = cfl_list_entry(sample_head, struct cmt_metric, _head); + normalize_histogram_sample(sample, histogram->buckets->count + 1); + } + } + + return 0; +} diff --git a/lib/cmetrics/tests/cat.c b/lib/cmetrics/tests/cat.c index 63852e623a0..b2408cb5cc4 100644 --- a/lib/cmetrics/tests/cat.c +++ b/lib/cmetrics/tests/cat.c @@ -28,6 +28,9 @@ #include #include #include +#include + +#include #include "cmt_tests.h" @@ -1083,7 +1086,200 @@ void test_cat_static_labels_empty() cmt_destroy(dst); } +static struct cmt *create_otlp_gauge_context(char *label, double value, uint64_t flags) +{ + struct cmt *cmt; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + + cmt = cmt_create(); + if (cmt == NULL) { + return NULL; + } + + gauge = cmt_gauge_create(cmt, "", "", "otlp_gauge", "otlp gauge", + 1, (char *[]) {"key"}); + if (gauge == NULL || + cmt_gauge_set(gauge, 100, value, 1, (char *[]) {label}) != 0) { + cmt_destroy(cmt); + return NULL; + } + + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata == NULL || + cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", flags) != 0) { + cmt_destroy(cmt); + return NULL; + } + + return cmt; +} + +static uint64_t sample_flags(struct cmt_metric *sample) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(sample->otlp_metadata, "flags"); + if (value == NULL) { + return UINT64_MAX; + } + if (value->type == CFL_VARIANT_UINT) { + return value->data.as_uint64; + } + if (value->type == CFL_VARIANT_INT) { + return (uint64_t) value->data.as_int64; + } + return UINT64_MAX; +} + +/* Samples carrying OTLP metadata must merge by labels, never duplicate. */ +void test_cat_otlp_metadata_samples_merge_by_labels() +{ + int ret; + struct cmt *first; + struct cmt *second; + struct cmt *other; + struct cmt *final; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cmt_map_label *label; + struct cfl_list *head; + int found_v; + int found_w; + + first = create_otlp_gauge_context("v", 1.0, 1); + second = create_otlp_gauge_context("v", 2.0, 2); + other = create_otlp_gauge_context("w", 3.0, 3); + final = cmt_create(); + TEST_ASSERT(first != NULL && second != NULL && other != NULL && final != NULL); + + ret = cmt_cat(final, first); + TEST_CHECK(ret == 0); + ret = cmt_cat(final, second); + TEST_CHECK(ret == 0); + ret = cmt_cat(final, second); + TEST_CHECK(ret == 0); + + TEST_CHECK(cfl_list_size(&final->gauges) == 1); + gauge = cfl_list_entry_first(&final->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 1); + + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_CHECK(cmt_metric_get_value(sample) == 2.0); + TEST_CHECK(sample_flags(sample) == 2); + + /* The merged sample remains reachable through the label index. */ + ret = cmt_gauge_set(gauge, 200, 5.0, 1, (char *[]) {"v"}); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 1); + TEST_CHECK(cmt_metric_get_value(sample) == 5.0); + + /* A distinct label set is still appended. */ + ret = cmt_cat(final, other); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 2); + + found_v = 0; + found_w = 0; + cfl_list_foreach(head, &gauge->map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + label = cfl_list_entry_first(&sample->labels, struct cmt_map_label, _head); + if (strcmp(label->name, "v") == 0) { + found_v++; + TEST_CHECK(sample_flags(sample) == 2); + } + else if (strcmp(label->name, "w") == 0) { + found_w++; + TEST_CHECK(sample_flags(sample) == 3); + TEST_CHECK(cmt_metric_get_value(sample) == 3.0); + } + } + TEST_CHECK(found_v == 1); + TEST_CHECK(found_w == 1); + + cmt_destroy(first); + cmt_destroy(second); + cmt_destroy(other); + cmt_destroy(final); +} + +/* Distinct OTLP data points of one series (different time windows) are kept, + * but concatenating them again must not duplicate either of them. */ +void test_cat_otlp_metadata_distinct_points_kept() +{ + int ret; + int pass; + struct cmt *source; + struct cmt *final; + struct cmt_gauge *gauge; + struct cmt_metric *first; + struct cmt_metric *second; + struct cmt_map_label *label; + struct cfl_list *head; + uint64_t timestamps; + + source = create_otlp_gauge_context("v", 1.0, 1); + TEST_ASSERT(source != NULL); + + /* Add a second data point of the same series at a later time, the way + * the OTLP decoder appends independent points. */ + gauge = cfl_list_entry_first(&source->gauges, struct cmt_gauge, _head); + first = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + second = calloc(1, sizeof(struct cmt_metric)); + TEST_ASSERT(second != NULL); + cfl_list_init(&second->labels); + cfl_list_init(&second->_hash_head); + second->map = gauge->map; + second->hash = first->hash; + cfl_list_add(&second->_head, &gauge->map->metrics); + TEST_ASSERT(cmt_map_metric_get(&gauge->opts, gauge->map, 1, (char *[]) {"v"}, + CMT_FALSE) == first); + label = calloc(1, sizeof(struct cmt_map_label)); + TEST_ASSERT(label != NULL); + label->name = cfl_sds_create("v"); + TEST_ASSERT(label->name != NULL); + cfl_list_add(&label->_head, &second->labels); + cmt_metric_set_double(second, 200, 2.0); + second->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(second->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(second->otlp_metadata, "flags", 2) == 0); + + final = cmt_create(); + TEST_ASSERT(final != NULL); + + for (pass = 0; pass < 2; pass++) { + ret = cmt_cat(final, source); + TEST_CHECK(ret == 0); + + gauge = cfl_list_entry_first(&final->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 2); + + timestamps = 0; + cfl_list_foreach(head, &gauge->map->metrics) { + first = cfl_list_entry(head, struct cmt_metric, _head); + timestamps += cmt_metric_get_timestamp(first); + if (cmt_metric_get_timestamp(first) == 100) { + TEST_CHECK(sample_flags(first) == 1); + TEST_CHECK(cmt_metric_get_value(first) == 1.0); + } + else { + TEST_CHECK(sample_flags(first) == 2); + TEST_CHECK(cmt_metric_get_value(first) == 2.0); + } + } + TEST_CHECK(timestamps == 300); + } + + cmt_destroy(source); + cmt_destroy(final); +} + TEST_LIST = { + {"cat_otlp_metadata_distinct_points_kept", test_cat_otlp_metadata_distinct_points_kept}, + {"cat_otlp_metadata_samples_merge_by_labels", test_cat_otlp_metadata_samples_merge_by_labels}, {"cat_static_labels_source_dynamic_conflict", test_cat_static_labels_source_dynamic_conflict}, {"cat_static_labels_dynamic_conflict", test_cat_static_labels_dynamic_conflict}, {"cat_static_labels_chain", test_cat_static_labels_chain}, diff --git a/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c new file mode 100644 index 00000000000..e598dc275c6 --- /dev/null +++ b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.c @@ -0,0 +1,39 @@ +/* + * Generator for histogram-cumulative-v5.1.0.msgpack, a fixture written by the + * bundled CMetrics library of Fluent Bit v5.1.0. It holds the native layout + * that release used for histograms built with cmt_histogram_observe() or + * decoded from Prometheus: cumulative buckets [1, 3, 5, 6], bounds + * [0.1, 0.5, 1] and count 6, once as the static sample and once labeled + * path="/a". The chunk is unmarked; readers infer the layout per sample. + * + * Build it against the headers and static libraries of a v5.1.0 checkout and + * redirect stdout to the fixture. The fixture is versioned on purpose: tests + * must never regenerate it with the current encoder. + */ +#include +#include +#include +#include +#include +int main(void) +{ + struct cmt *ctx = cmt_create(); + struct cmt_histogram *h; + char *keys[] = {"path"}; + char *vals[] = {"/a"}; + double samples[] = {0.05, 0.2, 0.3, 0.7, 0.8, 2.0}; + char *buffer; + size_t size; + int i; + h = cmt_histogram_create(ctx, "", "", "req_seconds", "request latency", + cmt_histogram_buckets_create(3, 0.1, 0.5, 1.0), 1, keys); + for (i = 0; i < 6; i++) { + cmt_histogram_observe(h, 100, samples[i], 1, vals); + cmt_histogram_observe(h, 100, samples[i], 0, NULL); + } + if (cmt_encode_msgpack_create(ctx, &buffer, &size) != 0) { return 1; } + fwrite(buffer, 1, size, stdout); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + return 0; +} diff --git a/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.msgpack b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.msgpack new file mode 100644 index 00000000000..7384021aed0 Binary files /dev/null and b/lib/cmetrics/tests/data/histogram-cumulative-v5.1.0.msgpack differ diff --git a/lib/cmetrics/tests/data/histogram-v5.1.0.c b/lib/cmetrics/tests/data/histogram-v5.1.0.c new file mode 100644 index 00000000000..2a9d8fc1fd2 --- /dev/null +++ b/lib/cmetrics/tests/data/histogram-v5.1.0.c @@ -0,0 +1,33 @@ +/* + * Generator for histogram-v5.1.0.msgpack, a fixture written by the bundled + * CMetrics library of Fluent Bit v5.1.0. It holds an OTLP-style histogram with + * independent bucket counts [5, 1, 3], count 9, sum 12 and bounds [1, 2]; that + * release stored OTLP-decoded histograms in this representation and wrote no + * bucket-layout marker. + * + * Build it against the headers and static libraries of a v5.1.0 checkout and + * redirect stdout to the fixture. The fixture is versioned on purpose: tests + * must never regenerate it with the current encoder. + */ +#include +#include +#include +#include +#include +int main(void) +{ + struct cmt *ctx = cmt_create(); + struct cmt_histogram *h; + uint64_t buckets[] = {5, 1, 3}; + char *buffer; + size_t size; + h = cmt_histogram_create(ctx, "", "", "legacy_histogram", "legacy", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + h->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + cmt_histogram_set_default(h, 100, buckets, 12.0, 9, 0, NULL); + if (cmt_encode_msgpack_create(ctx, &buffer, &size) != 0) { return 1; } + fwrite(buffer, 1, size, stdout); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + return 0; +} diff --git a/lib/cmetrics/tests/data/histogram-v5.1.0.msgpack b/lib/cmetrics/tests/data/histogram-v5.1.0.msgpack new file mode 100644 index 00000000000..014606de3a1 Binary files /dev/null and b/lib/cmetrics/tests/data/histogram-v5.1.0.msgpack differ diff --git a/lib/cmetrics/tests/msgpack_abi.c b/lib/cmetrics/tests/msgpack_abi.c index c9d5d13e318..d35fd47159e 100644 --- a/lib/cmetrics/tests/msgpack_abi.c +++ b/lib/cmetrics/tests/msgpack_abi.c @@ -347,7 +347,228 @@ void test_msgpack_abi_histogram_buckets_reject_mismatch() cmt_destroy(cmt); } +static struct cmt *create_static_gauge_with_metadata(void) +{ + struct cmt *source; + struct cmt_gauge *gauge; + struct cfl_kvlist *metadata; + + source = cmt_create(); + if (source == NULL) { + return NULL; + } + + gauge = cmt_gauge_create(source, "", "", "windows", "windows", 0, NULL); + if (gauge == NULL || cmt_gauge_set(gauge, 1, 1.0, 0, NULL) != 0) { + cmt_destroy(source); + return NULL; + } + + metadata = cfl_kvlist_create(); + gauge->map->metric.otlp_metadata = metadata; + if (metadata == NULL || + cfl_kvlist_insert_uint64(metadata, "flags", 7) != 0 || + cfl_kvlist_insert_string(metadata, "text", "value") != 0 || + cfl_kvlist_insert_bytes(metadata, "binary", "\0\xff", 2, CFL_FALSE) != 0) { + cmt_destroy(source); + return NULL; + } + + return source; +} + +static void test_point_metadata_and_unlabeled_windows(void) +{ + struct cmt *source; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *result; + struct cmt_metric *point; + struct cfl_variant *value; + char *buffer; + size_t size; + size_t offset; + int ret; + + /* Typed static point metadata survives a round trip. */ + source = create_static_gauge_with_metadata(); + TEST_ASSERT(source != NULL); + TEST_CHECK(cmt_encode_msgpack_create(source, &buffer, &size) == 0); + offset = 0; + ret = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_ASSERT(ret == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(offset == size); + result = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(result != NULL); + TEST_CHECK(cmt_metric_get_timestamp(&result->map->metric) == 1); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "flags"); + TEST_ASSERT(value != NULL); + TEST_CHECK((value->type == CFL_VARIANT_UINT && value->data.as_uint64 == 7) || + (value->type == CFL_VARIANT_INT && value->data.as_int64 == 7)); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "text"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->type == CFL_VARIANT_STRING && cfl_variant_size_get(value) == 5); + TEST_CHECK(memcmp(value->data.as_string, "value", 5) == 0); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "binary"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->type == CFL_VARIANT_BYTES && cfl_variant_size_get(value) == 2); + TEST_CHECK(memcmp(value->data.as_bytes, "\0\xff", 2) == 0); + TEST_CHECK(cfl_list_size(&result->map->metrics) == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(source); + + /* A further label-less point is the same series: like released readers, + * it replaces the static metric (last one wins) instead of creating a + * duplicate series. Its point metadata no longer lines up with the + * sidecar and is skipped, while the values and the chunk survive. */ + source = create_static_gauge_with_metadata(); + TEST_ASSERT(source != NULL); + gauge = cfl_list_entry_first(&source->gauges, struct cmt_gauge, _head); + point = calloc(1, sizeof(*point)); + TEST_ASSERT(point != NULL); + cfl_list_init(&point->labels); + point->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(point->otlp_metadata != NULL); + TEST_CHECK(cfl_kvlist_insert_uint64(point->otlp_metadata, "flags", 3) == 0); + cmt_metric_set_double(point, 2, 2.0); + cfl_list_add(&point->_head, &gauge->map->metrics); + TEST_CHECK(cmt_encode_msgpack_create(source, &buffer, &size) == 0); + offset = 0; + ret = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_ASSERT(ret == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(offset == size); + result = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(result != NULL); + TEST_CHECK(cfl_list_size(&result->map->metrics) == 0); + TEST_CHECK(cmt_metric_get_timestamp(&result->map->metric) == 2); + TEST_CHECK(cmt_metric_get_value(&result->map->metric) == 2.0); + TEST_CHECK(result->map->metric.otlp_metadata == NULL); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(source); +} + +static void test_index_resize_preserves_family_metadata(void) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cfl_variant *value; + char label[32]; + char *values[] = {label}; + char *keys[] = {"series"}; + size_t index; + char *buffer; + size_t size; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "resize", "resize", 1, keys); + TEST_ASSERT(gauge != NULL); + gauge->map->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(gauge->map->otlp_metadata, "family", "retained") == 0); + for (index = 0; index < 300; index++) { + snprintf(label, sizeof(label), "%zu", index); + TEST_ASSERT(cmt_gauge_set(gauge, 100, index, 1, values) == 0); + } + TEST_CHECK(gauge->map->metric_bucket_count > 64); + value = cfl_kvlist_fetch(gauge->map->otlp_metadata, "family"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_STRING); + if (value != NULL && value->type == CFL_VARIANT_STRING) { + TEST_CHECK(strcmp(value->data.as_string, "retained") == 0); + } + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void test_stale_metadata_sidecar_does_not_drop_chunk(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_array *sidecar; + char *buffer; + size_t size; + size_t offset = 0; + int result; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "retained", "retained", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + /* An intermediary removed a family without updating the optional sidecar. */ + sidecar = cfl_array_create(0); + TEST_ASSERT(sidecar != NULL); + TEST_ASSERT(cfl_kvlist_insert_array(ctx->internal_metadata, + "__cmetrics_native_metadata_v1", sidecar) == 0); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + if (decoded != NULL) { + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 42); + TEST_CHECK(cfl_kvlist_fetch(decoded->internal_metadata, + "__cmetrics_native_metadata_v1") == NULL); + cmt_destroy(decoded); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void test_invalid_optional_metadata_does_not_drop_chunk(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_array *sidecar; + struct cfl_array *points; + struct cfl_kvlist *family; + char *buffer; + size_t size; + size_t offset = 0; + int result; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "retained", "retained", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + sidecar = cfl_array_create(1); + points = cfl_array_create(1); + family = cfl_kvlist_create(); + TEST_ASSERT(sidecar != NULL && points != NULL && family != NULL); + TEST_ASSERT(cfl_array_append_null(points) == 0); + TEST_ASSERT(cfl_kvlist_insert_array(family, "points", points) == 0); + TEST_ASSERT(cfl_kvlist_insert_bool(family, "allow_reset", CFL_FALSE) == 0); + TEST_ASSERT(cfl_kvlist_insert_bytes(family, "family", "\xc1", 1, CFL_FALSE) == 0); + TEST_ASSERT(cfl_array_append_kvlist(sidecar, family) == 0); + TEST_ASSERT(cfl_kvlist_insert_array(ctx->internal_metadata, + "__cmetrics_native_metadata_v1", sidecar) == 0); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + if (decoded != NULL) { + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 42); + TEST_CHECK(gauge->map->otlp_metadata == NULL); + cmt_destroy(decoded); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + TEST_LIST = { + {"invalid_optional_metadata_retains_chunk", test_invalid_optional_metadata_does_not_drop_chunk}, + {"stale_metadata_sidecar_does_not_drop_chunk", test_stale_metadata_sidecar_does_not_drop_chunk}, + {"index_resize_preserves_family_metadata", test_index_resize_preserves_family_metadata}, + {"point_metadata_and_unlabeled_windows", test_point_metadata_and_unlabeled_windows}, {"msgpack_abi_legacy_value_only_decode", test_msgpack_abi_legacy_value_only_decode}, {"msgpack_abi_typed_int64_decode", test_msgpack_abi_typed_int64_decode}, {"msgpack_abi_summary_quantiles_reject_mismatch", test_msgpack_abi_summary_quantiles_reject_mismatch}, diff --git a/lib/cmetrics/tests/msgpack_temporality.c b/lib/cmetrics/tests/msgpack_temporality.c index b4a36e94aa0..e161aa88ed0 100644 --- a/lib/cmetrics/tests/msgpack_temporality.c +++ b/lib/cmetrics/tests/msgpack_temporality.c @@ -37,6 +37,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -174,6 +178,7 @@ static void test_counter_temporality_still_survives_msgpack_roundtrip() TEST_CHECK(counter != NULL); counter->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + counter->allow_reset = CMT_TRUE; offset = 0; result = cmt_encode_msgpack_create(cmt, &encoded_buffer, &encoded_size); @@ -191,6 +196,7 @@ static void test_counter_temporality_still_survives_msgpack_roundtrip() TEST_CHECK(decoded_counter != NULL); if (decoded_counter != NULL) { + TEST_CHECK(decoded_counter->allow_reset == CMT_TRUE); TEST_CHECK(decoded_counter->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); } @@ -202,7 +208,295 @@ static void test_counter_temporality_still_survives_msgpack_roundtrip() cmt_destroy(cmt); } +static void test_legacy_histogram_bucket_semantics() +{ + cfl_sds_t fixture; + struct cmt *decoded; + struct cmt_histogram *histogram; + size_t offset = 0; + + fixture = read_file(CMT_TESTS_DATA_PATH "/histogram-v5.1.0.msgpack"); + TEST_ASSERT(fixture != NULL); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, fixture, + cfl_sds_len(fixture), &offset) == 0); + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + /* Legacy OTLP chunks contain independent bucket counts [5,1,3]. */ + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 0) == 5); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 1) == 6); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 2) == 9); + TEST_CHECK(offset == cfl_sds_len(fixture)); + cmt_destroy(decoded); + cfl_sds_destroy(fixture); +} + +static int buffer_contains(char *buffer, size_t size, const void *needle, size_t length) +{ + size_t index; + + for (index = 0; index + length <= size; index++) { + if (memcmp(buffer + index, needle, length) == 0) { + return CMT_TRUE; + } + } + + return CMT_FALSE; +} + +static void check_histogram_sample(struct cmt_metric *sample, uint64_t *expected, + size_t count) +{ + size_t index; + + for (index = 0; index < count; index++) { + TEST_CHECK(cmt_metric_hist_get_value(sample, index) == expected[index]); + TEST_MSG("bucket %zu: expected %" PRIu64 ", got %" PRIu64, index, + expected[index], cmt_metric_hist_get_value(sample, index)); + } +} + +/* Fluent Bit v5.1.0 natively writes cumulative (Prometheus-style) buckets. */ +static void test_legacy_cumulative_histogram_fixture() +{ + cfl_sds_t fixture; + struct cmt *decoded; + struct cmt_histogram *histogram; + struct cmt_metric *sample; + size_t offset = 0; + uint64_t cumulative[] = {1, 3, 5, 6}; + uint64_t independent[] = {1, 2, 2, 1}; + size_t index; + + fixture = read_file(CMT_TESTS_DATA_PATH "/histogram-cumulative-v5.1.0.msgpack"); + TEST_ASSERT(fixture != NULL); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, fixture, + cfl_sds_len(fixture), &offset) == 0); + TEST_CHECK(offset == cfl_sds_len(fixture)); + + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + TEST_ASSERT(histogram != NULL); + TEST_CHECK(histogram->map->metric_static_set == CMT_TRUE); + check_histogram_sample(&histogram->map->metric, cumulative, 4); + + TEST_ASSERT(cfl_list_size(&histogram->map->metrics) == 1); + sample = cfl_list_entry_first(&histogram->map->metrics, struct cmt_metric, _head); + check_histogram_sample(sample, cumulative, 4); + TEST_CHECK(cmt_metric_hist_get_count_value(sample) == 6); + + for (index = 0; index < 4; index++) { + TEST_CHECK(cmt_metric_hist_get_bucket_count(sample, index) == independent[index]); + } + + cmt_destroy(decoded); + cfl_sds_destroy(fixture); +} + +/* The wire must stay byte-compatible with v5.1.0 readers: cumulative and + * unmarked, with or without the OTLP metadata sidecar. */ +static void test_histogram_wire_stays_cumulative() +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_histogram *histogram; + char *buffer; + size_t size; + size_t offset; + int pass; + uint64_t counts[] = {5, 6, 9}; + const unsigned char expected[] = {0xa7, 'b','u','c','k','e','t','s',0x93,5,6,9}; + const char marker[] = "__cmetrics_histogram_buckets"; + + for (pass = 0; pass < 2; pass++) { + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + histogram = cmt_histogram_create(ctx, "", "", "hist", "hist", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_set_default(histogram, 100, counts, 12.0, 9, 0, NULL) == 0); + + if (pass == 1) { + /* Force the OTLP metadata sidecar path. */ + histogram->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(histogram->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(histogram->map->metric.otlp_metadata, + "flags", 1) == 0); + } + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + TEST_CHECK(buffer_contains(buffer, size, expected, sizeof(expected))); + TEST_CHECK(!buffer_contains(buffer, size, marker, sizeof(marker) - 1)); + + offset = 0; + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + check_histogram_sample(&histogram->map->metric, counts, 3); + if (pass == 1) { + TEST_CHECK(histogram->map->metric.otlp_metadata != NULL); + } + + cmt_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + } +} + +/* Encode a histogram and splice a bucket marker into the internal metadata + * map, as unreleased writers did; the current encoder never emits it. */ +static int encode_with_marker(uint64_t *counts, uint64_t total, + const char *marker, size_t marker_length, + char **buffer, size_t *size) +{ + struct cmt *ctx; + struct cmt_histogram *histogram; + char *encoded; + char *spliced; + size_t encoded_size; + size_t index; + size_t position; + size_t key_length; + const char header[] = "\xa8" "cmetrics" "\x80"; + const char key[] = "__cmetrics_histogram_buckets"; + + ctx = cmt_create(); + if (ctx == NULL) { + return -1; + } + + histogram = cmt_histogram_create(ctx, "", "", "hist", "hist", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + if (histogram == NULL || + cmt_histogram_set_default(histogram, 100, counts, 12.0, total, 0, NULL) != 0 || + cmt_encode_msgpack_create(ctx, &encoded, &encoded_size) != 0) { + cmt_destroy(ctx); + return -1; + } + cmt_destroy(ctx); + + position = encoded_size; + for (index = 0; index + sizeof(header) - 1 <= encoded_size; index++) { + if (memcmp(encoded + index, header, sizeof(header) - 1) == 0) { + position = index + sizeof(header) - 2; + break; + } + } + if (position == encoded_size) { + cmt_encode_msgpack_destroy(encoded); + return -1; + } + + key_length = sizeof(key) - 1; + spliced = malloc(encoded_size + 2 + key_length + marker_length); + if (spliced == NULL) { + cmt_encode_msgpack_destroy(encoded); + return -1; + } + + /* fixmap(1) { fixstr(key): marker } replaces the empty fixmap. */ + memcpy(spliced, encoded, position); + index = position; + spliced[index++] = (char) 0x81; + spliced[index++] = (char) (0xa0 | key_length); + memcpy(spliced + index, key, key_length); + index += key_length; + memcpy(spliced + index, marker, marker_length); + index += marker_length; + memcpy(spliced + index, encoded + position + 1, encoded_size - position - 1); + index += encoded_size - position - 1; + + cmt_encode_msgpack_destroy(encoded); + *buffer = spliced; + *size = index; + + return 0; +} + +/* Bucket markers written by unreleased builds are advisory: unknown or + * mistyped values must not drop the chunk, and a marker can not force a + * wrong interpretation of the buckets. */ +static void test_histogram_marker_is_ignored() +{ + struct cmt *decoded; + struct cmt_histogram *histogram; + char *buffer; + size_t size; + size_t offset; + size_t index; + int ret; + uint64_t cumulative[] = {5, 6, 9}; + uint64_t independent[] = {5, 1, 3}; + struct { + const char *marker; + size_t length; + uint64_t *counts; + } cases[] = { + {"\xa5" "bogus", 6, cumulative}, + {"\x07", 1, cumulative}, + {"\xc0", 1, cumulative}, + {"\xaa" "per_bucket", 11, cumulative}, + /* spoofed: "cumulative" claimed on independent counts */ + {"\xaa" "cumulative", 11, independent}, + {"\xaa" "cumulative", 11, cumulative}, + {"\xaa" "per_bucket", 11, independent}, + }; + + for (index = 0; index < sizeof(cases) / sizeof(cases[0]); index++) { + buffer = NULL; + size = 0; + ret = encode_with_marker(cases[index].counts, 9, cases[index].marker, + cases[index].length, &buffer, &size); + TEST_ASSERT(ret == 0); + + offset = 0; + decoded = NULL; + ret = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(ret == CMT_DECODE_MSGPACK_SUCCESS); + TEST_MSG("case %zu: decode returned %d", index, ret); + TEST_CHECK(offset == size); + + if (decoded != NULL) { + histogram = cfl_list_entry_first(&decoded->histograms, + struct cmt_histogram, _head); + check_histogram_sample(&histogram->map->metric, cumulative, 3); + TEST_CHECK(cfl_kvlist_fetch(decoded->internal_metadata, + "__cmetrics_histogram_buckets") == NULL); + cmt_destroy(decoded); + } + + free(buffer); + } +} + +/* Samples whose shape is not consistent with either layout are untouched. */ +static void test_histogram_inconsistent_buckets_untouched() +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_histogram *histogram; + char *buffer; + size_t size; + size_t offset = 0; + uint64_t counts[] = {4, 2, 8}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + histogram = cmt_histogram_create(ctx, "", "", "hist", "hist", + cmt_histogram_buckets_create(2, 1.0, 2.0), 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_set_default(histogram, 100, counts, 1.0, 11, 0, NULL) == 0); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + histogram = cfl_list_entry_first(&decoded->histograms, struct cmt_histogram, _head); + check_histogram_sample(&histogram->map->metric, counts, 3); + cmt_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + TEST_LIST = { + {"legacy_histogram_bucket_semantics", test_legacy_histogram_bucket_semantics}, + {"legacy_cumulative_histogram_fixture", test_legacy_cumulative_histogram_fixture}, + {"histogram_wire_stays_cumulative", test_histogram_wire_stays_cumulative}, + {"histogram_marker_is_ignored", test_histogram_marker_is_ignored}, + {"histogram_inconsistent_buckets_untouched", test_histogram_inconsistent_buckets_untouched}, {"histogram_temporality_survives_msgpack_roundtrip", test_histogram_temporality_survives_msgpack_roundtrip}, {"exp_histogram_temporality_survives_msgpack_roundtrip", diff --git a/lib/cmetrics/tests/opentelemetry.c b/lib/cmetrics/tests/opentelemetry.c index f225a7fcfb9..0bf8221ec37 100644 --- a/lib/cmetrics/tests/opentelemetry.c +++ b/lib/cmetrics/tests/opentelemetry.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -25,6 +26,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -1856,11 +1860,8 @@ static void test_opentelemetry_zero_length_bytes_attribute(void) attribute = roundtrip_dp->attributes[0]; TEST_CHECK(strcmp(attribute->key, "empty_bytes") == 0); TEST_CHECK(attribute->value->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE); - TEST_CHECK(attribute->value->string_value != NULL); - if (attribute->value->string_value != NULL) { - TEST_CHECK(attribute->value->string_value[0] == '\0'); - } + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE); + TEST_CHECK(attribute->value->bytes_value.len == 0); } } } @@ -2813,7 +2814,418 @@ static void test_opentelemetry_summary_layout_mismatch(void) cfl_sds_destroy(payload); } +static void test_otlp_unsigned_attribute_preserves_signed_range(void) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cfl_kvlist *attrs; + cfl_sds_t wire; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *point; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "uint_attributes", "uint attributes", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + gauge->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->metric.otlp_metadata != NULL); + attrs = cfl_kvlist_create(); + TEST_ASSERT(attrs != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(attrs, "small", 3) == 0); + TEST_ASSERT(cfl_kvlist_insert_uint64(attrs, "maximum", INT64_MAX) == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->metric.otlp_metadata, "attributes", attrs) == 0); + TEST_ASSERT(cmt_otlp_capture_labels(gauge->map, &gauge->map->metric) == 0); + wire = cmt_encode_opentelemetry_create(ctx); + TEST_CHECK(wire != NULL); + if (wire != NULL) { + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + point = request->resource_metrics[0]->scope_metrics[0]->metrics[0]->gauge->data_points[0]; + TEST_ASSERT(point->n_attributes == 2); + TEST_CHECK(point->attributes[0]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE); + TEST_CHECK(point->attributes[0]->value->int_value == 3); + TEST_CHECK(point->attributes[1]->value->int_value == INT64_MAX); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + } + cmt_destroy(ctx); +} + +/* Metadata deeper than the msgpack nesting limit stays usable. */ +static void test_otlp_metadata_deep_nesting(void) +{ + int depth; + int found = 0; + struct cfl_list *head; + struct cmt_gauge *entry; + char *buffer; + size_t size; + size_t offset; + struct cmt *ctx; + struct cmt *copy; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *shallow; + struct cmt_metric *sample; + struct cfl_kvlist *inner; + struct cfl_kvlist *outer; + char *labels[] = {"a"}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "deep", "deep", 1, (char *[]) {"k"}); + shallow = cmt_gauge_create(ctx, "", "", "shallow", "shallow", 0, NULL); + TEST_ASSERT(gauge != NULL && shallow != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 1000, 7, 1, labels) == 0); + TEST_ASSERT(cmt_gauge_set(shallow, 1000, 9, 0, NULL) == 0); + + /* a point whose metadata nests 40 maps deep */ + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + inner = cfl_kvlist_create(); + TEST_ASSERT(inner != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(inner, "leaf", "value") == 0); + for (depth = 0; depth < 40; depth++) { + outer = cfl_kvlist_create(); + TEST_ASSERT(outer != NULL); + TEST_ASSERT(cfl_kvlist_insert_kvlist(outer, "nested", inner) == 0); + inner = outer; + } + sample->otlp_metadata = inner; + + shallow->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(shallow->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(shallow->map->metric.otlp_metadata, + "start_time_unix_nano", 500) == 0); + + /* copying a context copies the deep metadata too */ + copy = cmt_create(); + TEST_ASSERT(copy != NULL); + TEST_CHECK(cmt_cat(copy, ctx) == 0); + gauge = cfl_list_entry_first(©->gauges, struct cmt_gauge, _head); + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_CHECK(sample->otlp_metadata != NULL && + cfl_kvlist_fetch(sample->otlp_metadata, "nested") != NULL); + cmt_destroy(copy); + + /* a msgpack round trip skips only the point it cannot read back */ + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + offset = 0; + decoded = NULL; + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + cfl_list_foreach(head, &decoded->gauges) { + entry = cfl_list_entry(head, struct cmt_gauge, _head); + if (strcmp(entry->opts.name, "shallow") == 0) { + found++; + TEST_CHECK(entry->map->metric.otlp_metadata != NULL && + cfl_kvlist_fetch(entry->map->metric.otlp_metadata, + "start_time_unix_nano") != NULL); + } + else if (strcmp(entry->opts.name, "deep") == 0) { + found++; + sample = cfl_list_entry_first(&entry->map->metrics, struct cmt_metric, _head); + TEST_CHECK(sample->otlp_metadata == NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 7); + } + } + TEST_CHECK(found == 2); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Older readers find point metadata in the external tree, keyed by series. */ +static void test_otlp_metadata_legacy_external_tree(void) +{ + char key[64]; + char *buffer = NULL; + size_t size = 0; + mpack_writer_t writer; + mpack_reader_t reader; + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *family; + struct cfl_kvlist *metadata; + struct cfl_kvlist *external = NULL; + struct cfl_variant *node; + struct cfl_variant *point; + char *labels[] = {"a"}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "legacy", "legacy", 1, (char *[]) {"k"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 1000, 7, 1, labels) == 0); + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_ASSERT(sample != NULL); + + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "start_time_unix_nano", 500) == 0); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", 1) == 0); + TEST_ASSERT(cfl_kvlist_insert_string(sample->otlp_metadata, "number_value_case", "double") == 0); + TEST_ASSERT(cmt_otlp_capture_labels(gauge->map, sample) == 0); + + gauge->map->otlp_metadata = cfl_kvlist_create(); + metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(metadata, "unit.source", "otlp") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", metadata) == 0); + + mpack_writer_init_growable(&writer, &buffer, &size); + TEST_ASSERT(cmt_otlp_metadata_pack_external(&writer, ctx) == 0); + TEST_ASSERT(mpack_writer_destroy(&writer) == mpack_ok); + mpack_reader_init_data(&reader, buffer, size); + TEST_ASSERT(unpack_cfl_kvlist(&reader, &external) == 0); + mpack_reader_destroy(&reader); + free(buffer); + + node = cfl_kvlist_fetch(external, "otlp"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(node->data.as_kvlist, "metrics"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(node->data.as_kvlist, "gauge"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(node->data.as_kvlist, "legacy"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + family = node->data.as_kvlist; + + node = cfl_kvlist_fetch(family, "metadata"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + TEST_CHECK(cfl_kvlist_fetch(node->data.as_kvlist, "unit.source") != NULL); + + node = cfl_kvlist_fetch(family, "datapoints"); + TEST_ASSERT(node != NULL && node->type == CFL_VARIANT_KVLIST); + snprintf(key, sizeof(key), "%" PRIx64 ":%" PRIu64, + sample->hash, cmt_metric_get_timestamp(sample)); + point = cfl_kvlist_fetch(node->data.as_kvlist, key); + TEST_ASSERT(point != NULL && point->type == CFL_VARIANT_KVLIST); + node = cfl_kvlist_fetch(point->data.as_kvlist, "start_time_unix_nano"); + TEST_CHECK(node != NULL && node->data.as_uint64 == 500); + node = cfl_kvlist_fetch(point->data.as_kvlist, "flags"); + TEST_CHECK(node != NULL && node->data.as_uint64 == 1); + /* only the keys older readers use are written */ + TEST_CHECK(cfl_kvlist_fetch(point->data.as_kvlist, "attributes") == NULL); + + /* the context itself is unchanged */ + TEST_CHECK(cfl_kvlist_fetch(ctx->external_metadata, "otlp") == NULL); + + cfl_kvlist_destroy(external); + cmt_destroy(ctx); +} + +/* Typed attributes without an OTLP form keep the label derived attributes. */ +static void test_otlp_unrepresentable_attribute_keeps_point(void) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cfl_kvlist *attrs; + cfl_sds_t wire; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "huge_attribute", "huge attribute", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 0, NULL) == 0); + gauge->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->metric.otlp_metadata != NULL); + attrs = cfl_kvlist_create(); + TEST_ASSERT(attrs != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(attrs, "huge", UINT64_MAX) == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->metric.otlp_metadata, "attributes", attrs) == 0); + TEST_ASSERT(cmt_otlp_capture_labels(gauge->map, &gauge->map->metric) == 0); + + wire = cmt_encode_opentelemetry_create(ctx); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + TEST_ASSERT(request->n_resource_metrics == 1); + metric = request->resource_metrics[0]->scope_metrics[0]->metrics[0]; + TEST_CHECK(strcmp(metric->name, "huge_attribute") == 0); + TEST_ASSERT(metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->as_double == 42); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_destroy(ctx); +} + +/* Repeated label-less points of one metric are the same series: the last one + * replaces the static sample, as released versions behave. */ +static cfl_sds_t generate_repeated_unlabeled_points_payload() +{ + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest request; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics resource_metrics; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics scope_metrics; + Opentelemetry__Proto__Metrics__V1__Metric gauge_metric; + Opentelemetry__Proto__Metrics__V1__Metric histogram_metric; + Opentelemetry__Proto__Metrics__V1__Gauge gauge; + Opentelemetry__Proto__Metrics__V1__Histogram histogram; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint number_points[2]; + Opentelemetry__Proto__Metrics__V1__HistogramDataPoint histogram_points[2]; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics *resource_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics *scope_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__Metric *metric_list[2]; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *number_point_list[2]; + Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *histogram_point_list[2]; + uint64_t first_buckets[] = {5, 1, 3}; + uint64_t second_buckets[] = {1, 1, 1}; + double bounds[] = {1.0, 2.0}; + size_t payload_size; + size_t index; + unsigned char *packed_payload; + cfl_sds_t payload; + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(&request); + opentelemetry__proto__metrics__v1__resource_metrics__init(&resource_metrics); + opentelemetry__proto__metrics__v1__scope_metrics__init(&scope_metrics); + opentelemetry__proto__metrics__v1__metric__init(&gauge_metric); + opentelemetry__proto__metrics__v1__metric__init(&histogram_metric); + opentelemetry__proto__metrics__v1__gauge__init(&gauge); + opentelemetry__proto__metrics__v1__histogram__init(&histogram); + + for (index = 0; index < 2; index++) { + opentelemetry__proto__metrics__v1__number_data_point__init(&number_points[index]); + number_points[index].time_unix_nano = index + 1; + number_points[index].flags = index; + number_points[index].value_case = + OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + number_points[index].as_double = index + 1.0; + number_point_list[index] = &number_points[index]; + + opentelemetry__proto__metrics__v1__histogram_data_point__init(&histogram_points[index]); + histogram_points[index].time_unix_nano = index + 1; + histogram_points[index].n_bucket_counts = 3; + histogram_points[index].bucket_counts = index == 0 ? first_buckets : second_buckets; + histogram_points[index].n_explicit_bounds = 2; + histogram_points[index].explicit_bounds = bounds; + histogram_points[index].count = index == 0 ? 9 : 3; + histogram_points[index].has_sum = 1; + histogram_points[index].sum = 1.0; + histogram_point_list[index] = &histogram_points[index]; + } + + gauge.n_data_points = 2; + gauge.data_points = number_point_list; + gauge_metric.name = "u"; + gauge_metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE; + gauge_metric.gauge = &gauge; + + histogram.n_data_points = 2; + histogram.data_points = histogram_point_list; + histogram.aggregation_temporality = + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE; + histogram_metric.name = "h"; + histogram_metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM; + histogram_metric.histogram = &histogram; + + metric_list[0] = &gauge_metric; + metric_list[1] = &histogram_metric; + scope_metrics.n_metrics = 2; + scope_metrics.metrics = metric_list; + + scope_metrics_list[0] = &scope_metrics; + resource_metrics.n_scope_metrics = 1; + resource_metrics.scope_metrics = scope_metrics_list; + + resource_metrics_list[0] = &resource_metrics; + request.n_resource_metrics = 1; + request.resource_metrics = resource_metrics_list; + + payload_size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(&request); + packed_payload = calloc(1, payload_size); + if (packed_payload == NULL) { + return NULL; + } + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(&request, + packed_payload); + + payload = cfl_sds_create_len((char *) packed_payload, payload_size); + free(packed_payload); + + return payload; +} + +static void test_opentelemetry_repeated_unlabeled_points_last_wins(void) +{ + cfl_sds_t payload; + cfl_sds_t encoded; + size_t offset; + int result; + struct cfl_list decoded_context_list; + struct cmt *decoded_context; + struct cmt_gauge *gauge; + struct cmt_histogram *histogram; + struct cfl_variant *flags; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + payload = generate_repeated_unlabeled_points_payload(); + TEST_ASSERT(payload != NULL); + + offset = 0; + result = cmt_decode_opentelemetry_create(&decoded_context_list, payload, + cfl_sds_len(payload), &offset); + TEST_ASSERT(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + + decoded_context = cfl_list_entry_first(&decoded_context_list, struct cmt, _head); + TEST_ASSERT(decoded_context != NULL); + + gauge = cfl_list_entry_first(&decoded_context->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 0); + TEST_CHECK(gauge->map->metric_static_set == CMT_TRUE); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 2.0); + TEST_CHECK(cmt_metric_get_timestamp(&gauge->map->metric) == 2); + flags = cfl_kvlist_fetch(gauge->map->metric.otlp_metadata, "flags"); + TEST_ASSERT(flags != NULL); + TEST_CHECK(flags->type == CFL_VARIANT_UINT && flags->data.as_uint64 == 1); + + histogram = cfl_list_entry_first(&decoded_context->histograms, struct cmt_histogram, _head); + TEST_ASSERT(histogram != NULL); + TEST_CHECK(cfl_list_size(&histogram->map->metrics) == 0); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 0) == 1); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 1) == 2); + TEST_CHECK(cmt_metric_hist_get_value(&histogram->map->metric, 2) == 3); + TEST_CHECK(cmt_metric_hist_get_count_value(&histogram->map->metric) == 3); + + encoded = cmt_encode_opentelemetry_create(decoded_context); + TEST_ASSERT(encoded != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(encoded), (const uint8_t *) encoded); + TEST_ASSERT(request != NULL); + + metric = find_otlp_metric_by_name(request, "u"); + TEST_ASSERT(metric != NULL && metric->gauge != NULL); + TEST_CHECK(metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->as_double == 2.0); + + metric = find_otlp_metric_by_name(request, "h"); + TEST_ASSERT(metric != NULL && metric->histogram != NULL); + TEST_CHECK(metric->histogram->n_data_points == 1); + TEST_CHECK(metric->histogram->data_points[0]->n_bucket_counts == 3); + TEST_CHECK(metric->histogram->data_points[0]->bucket_counts[0] == 1); + TEST_CHECK(metric->histogram->data_points[0]->bucket_counts[1] == 1); + TEST_CHECK(metric->histogram->data_points[0]->bucket_counts[2] == 1); + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(encoded); + cmt_decode_opentelemetry_destroy(&decoded_context_list); + cfl_sds_destroy(payload); +} + TEST_LIST = { + {"otlp_unrepresentable_attribute_keeps_point", test_otlp_unrepresentable_attribute_keeps_point}, + {"otlp_metadata_legacy_external_tree", test_otlp_metadata_legacy_external_tree}, + {"otlp_metadata_deep_nesting", test_otlp_metadata_deep_nesting}, + {"repeated_unlabeled_points_last_wins", test_opentelemetry_repeated_unlabeled_points_last_wins}, + {"otlp_unsigned_attribute_signed_range", test_otlp_unsigned_attribute_preserves_signed_range}, {"opentelemetry_api_full_roundtrip_with_msgpack", test_opentelemetry_api_full_roundtrip_with_msgpack}, {"opentelemetry_encode_multi_resource_scope_containers", test_opentelemetry_encode_multi_resource_scope_containers}, {"opentelemetry_encode_scope_attributes_cleanup", test_opentelemetry_encode_scope_attributes_cleanup}, diff --git a/lib/ctraces/src/ctr_decode_msgpack.c b/lib/ctraces/src/ctr_decode_msgpack.c index a77b932ca27..fc3de4a65bb 100644 --- a/lib/ctraces/src/ctr_decode_msgpack.c +++ b/lib/ctraces/src/ctr_decode_msgpack.c @@ -512,8 +512,9 @@ static int unpack_span_kind(mpack_reader_t *reader, size_t index, void *ctx) int result; result = ctr_mpack_consume_int32_tag(reader, &kind); - if (result == CTR_MPACK_SUCCESS && ctr_span_kind_set(context->span, kind) != 0) { - return CTR_MPACK_CORRUPT_INPUT_DATA_ERROR; + if (result == CTR_MPACK_SUCCESS) { + /* Match proto3 decoding: preserve unknown received enum values. */ + context->span->kind = kind; } return result; diff --git a/lib/ctraces/src/ctr_decode_opentelemetry.c b/lib/ctraces/src/ctr_decode_opentelemetry.c index 60ae7b07e29..8bb878e2ad0 100644 --- a/lib/ctraces/src/ctr_decode_opentelemetry.c +++ b/lib/ctraces/src/ctr_decode_opentelemetry.c @@ -774,10 +774,8 @@ int ctr_decode_opentelemetry_create(struct ctrace **out_ctr, ctr_span_set_trace_state(span, otel_span->trace_state, strlen(otel_span->trace_state)); } - if (ctr_span_kind_set(span, otel_span->kind) != 0) { - return decode_opentelemetry_error( - service_request, ctr, CTR_DECODE_OPENTELEMETRY_INVALID_PAYLOAD); - } + /* Preserve unknown proto3 enum values received on the wire. */ + span->kind = otel_span->kind; ctr_span_set_flags(span, otel_span->flags); ctr_span_start_ts(ctr, span, otel_span->start_time_unix_nano); ctr_span_end_ts(ctr, span, otel_span->end_time_unix_nano); diff --git a/lib/ctraces/src/ctr_encode_msgpack.c b/lib/ctraces/src/ctr_encode_msgpack.c index b20b3b00c4e..b2bf014cfbe 100644 --- a/lib/ctraces/src/ctr_encode_msgpack.c +++ b/lib/ctraces/src/ctr_encode_msgpack.c @@ -361,9 +361,18 @@ static void pack_span(mpack_writer_t *writer, struct ctrace_span *span) mpack_write_nil(writer); } - /* kind */ + /* + * kind: decoders before unknown kinds were preserved reject a whole + * context when the value is outside the known range, so unknown kinds + * are stored as unspecified. + */ mpack_write_cstr(writer, "kind"); - mpack_write_u32(writer, span->kind); + if (span->kind >= CTRACE_SPAN_UNSPECIFIED && span->kind <= CTRACE_SPAN_CONSUMER) { + mpack_write_u32(writer, span->kind); + } + else { + mpack_write_u32(writer, CTRACE_SPAN_UNSPECIFIED); + } /* start_time_unix_nano */ mpack_write_cstr(writer, "start_time_unix_nano"); diff --git a/lib/ctraces/src/ctr_encode_opentelemetry.c b/lib/ctraces/src/ctr_encode_opentelemetry.c index e56770fd089..5c6a114353d 100644 --- a/lib/ctraces/src/ctr_encode_opentelemetry.c +++ b/lib/ctraces/src/ctr_encode_opentelemetry.c @@ -658,26 +658,8 @@ static void otel_span_set_parent_span_id(Opentelemetry__Proto__Trace__V1__Span * static void otel_span_set_kind(Opentelemetry__Proto__Trace__V1__Span *otel_span, int kind) { - switch (kind) { - case CTRACE_SPAN_INTERNAL: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_INTERNAL; - break; - case CTRACE_SPAN_SERVER: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_SERVER; - break; - case CTRACE_SPAN_CLIENT: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_CLIENT; - break; - case CTRACE_SPAN_PRODUCER: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_PRODUCER; - break; - case CTRACE_SPAN_CONSUMER: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_CONSUMER; - break; - default: - otel_span->kind = OPENTELEMETRY__PROTO__TRACE__V1__SPAN__SPAN_KIND__SPAN_KIND_UNSPECIFIED; - break; - } + /* CTrace kind values match the OTLP enum. Preserve unknown proto3 values. */ + otel_span->kind = kind; } static void otel_span_set_start_time(Opentelemetry__Proto__Trace__V1__Span *span, diff --git a/lib/ctraces/tests/decoding.c b/lib/ctraces/tests/decoding.c index 04239b9245c..5e7cf9cc986 100644 --- a/lib/ctraces/tests/decoding.c +++ b/lib/ctraces/tests/decoding.c @@ -625,6 +625,55 @@ void test_msgpack_preserves_flags() ctr_destroy(ctx); } +void test_msgpack_stores_unknown_kind_as_unspecified() +{ + char *buffer; + size_t size; + size_t offset; + struct ctrace *ctx; + struct ctrace *decoded; + struct ctrace_resource_span *rs; + struct ctrace_scope_span *ss; + struct ctrace_span *span; + int kinds[] = {-1, INT32_MIN, 0, 5, INT32_MAX}; + size_t index; + + for (index = 0; index < sizeof(kinds) / sizeof(kinds[0]); index++) { + ctx = ctr_create(NULL); + TEST_ASSERT(ctx != NULL); + rs = ctr_resource_span_create(ctx); + TEST_ASSERT(rs != NULL); + ss = ctr_scope_span_create(rs); + TEST_ASSERT(ss != NULL); + span = ctr_span_create(ctx, ss, "unknown-kind", NULL); + TEST_ASSERT(span != NULL); + /* OTLP decoders keep unknown enum values, msgpack stores them as unspecified */ + span->kind = kinds[index]; + TEST_ASSERT(ctr_span_create(ctx, ss, "healthy-sibling", NULL) != NULL); + TEST_ASSERT(ctr_encode_msgpack_create(ctx, &buffer, &size) == 0); + offset = 0; + decoded = NULL; + TEST_CHECK(ctr_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + if (decoded != NULL) { + rs = cfl_list_entry(decoded->resource_spans.next, + struct ctrace_resource_span, _head); + ss = cfl_list_entry(rs->scope_spans.next, struct ctrace_scope_span, _head); + span = cfl_list_entry(ss->spans.next, struct ctrace_span, _head); + if (kinds[index] >= CTRACE_SPAN_UNSPECIFIED && kinds[index] <= CTRACE_SPAN_CONSUMER) { + TEST_CHECK(span->kind == kinds[index]); + } + else { + TEST_CHECK(span->kind == CTRACE_SPAN_UNSPECIFIED); + } + TEST_CHECK(cfl_list_size(&ss->spans) == 2); + TEST_CHECK(offset == size); + ctr_destroy(decoded); + } + ctr_encode_msgpack_destroy(buffer); + ctr_destroy(ctx); + } +} + void test_msgpack_invalid_offset() { char data[] = {0x80}; @@ -1013,6 +1062,7 @@ void test_simple_to_msgpack_and_back() TEST_LIST = { + {"msgpack_unknown_kind_unspecified", test_msgpack_stores_unknown_kind_as_unspecified}, {"cmt_simple_to_msgpack_and_back", test_simple_to_msgpack_and_back}, {"cmt_msgpack", test_msgpack_to_cmt}, {"empty_spans", test_msgpack_to_ctr_with_empty_spans}, diff --git a/lib/ctraces/tests/opentelemetry.c b/lib/ctraces/tests/opentelemetry.c index fee972b3982..1d8c249ae7b 100644 --- a/lib/ctraces/tests/opentelemetry.c +++ b/lib/ctraces/tests/opentelemetry.c @@ -670,7 +670,7 @@ void test_otlp_decode_span_empty_name() free(wire); } -void test_otlp_decode_invalid_kind() +void test_otlp_preserves_unknown_kind() { Opentelemetry__Proto__Trace__V1__Span span; uint8_t *wire; @@ -688,8 +688,12 @@ void test_otlp_decode_invalid_kind() offset = 0; decoded = NULL; ret = ctr_decode_opentelemetry_create(&decoded, (char *) wire, wire_len, &offset); - TEST_CHECK(ret != CTR_DECODE_OPENTELEMETRY_SUCCESS); - TEST_CHECK(decoded == NULL); + TEST_ASSERT(ret == CTR_DECODE_OPENTELEMETRY_SUCCESS); + TEST_ASSERT(decoded != NULL); + TEST_CHECK(cfl_list_entry_first(&decoded->span_list, struct ctrace_span, _head_global)->kind == 99); + TEST_CHECK(ctr_span_kind_set(cfl_list_entry_first(&decoded->span_list, + struct ctrace_span, _head_global), 99) == -1); + ctr_destroy(decoded); free(wire); } @@ -774,6 +778,6 @@ TEST_LIST = { {"otlp_decode_attribute_null_value", test_otlp_decode_attribute_null_value}, {"otlp_decode_nested_malformed_kv", test_otlp_decode_nested_malformed_kv}, {"otlp_decode_span_empty_name", test_otlp_decode_span_empty_name}, - {"otlp_decode_invalid_kind", test_otlp_decode_invalid_kind}, + {"otlp_preserves_unknown_kind", test_otlp_preserves_unknown_kind}, { 0 } }; diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 86e19800367..61b970e7c33 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -347,6 +347,9 @@ REGISTER_IN_PLUGIN("in_random") # ========== REGISTER_PROCESSOR_PLUGIN("processor_content_modifier") REGISTER_PROCESSOR_PLUGIN("processor_cumulative_to_delta") +if(FLB_LUAJIT AND FLB_YYJSON) + REGISTER_PROCESSOR_PLUGIN("processor_fts") +endif() REGISTER_PROCESSOR_PLUGIN("processor_labels") REGISTER_PROCESSOR_PLUGIN("processor_metrics_selector") REGISTER_PROCESSOR_PLUGIN("processor_opentelemetry_envelope") diff --git a/plugins/out_opentelemetry/opentelemetry_logs.c b/plugins/out_opentelemetry/opentelemetry_logs.c index e1366a995bb..f790a4fde7b 100644 --- a/plugins/out_opentelemetry/opentelemetry_logs.c +++ b/plugins/out_opentelemetry/opentelemetry_logs.c @@ -175,6 +175,32 @@ static msgpack_object *msgpack_map_get_object(msgpack_object_map *map, return &map->ptr[index].val; } +static uint32_t dropped_attribute_count(msgpack_object *object) +{ + msgpack_object *field; + + if (!object || object->type != MSGPACK_OBJECT_MAP) { + return 0; + } + + field = msgpack_map_get_object(&object->via.map, "dropped_attributes_count"); + if (field && field->type == MSGPACK_OBJECT_POSITIVE_INTEGER && + field->via.u64 <= UINT32_MAX) { + return (uint32_t) field->via.u64; + } + + return 0; +} + +static uint32_t group_dropped_attribute_count(msgpack_object *body, const char *kind) +{ + if (!body || body->type != MSGPACK_OBJECT_MAP) { + return 0; + } + + return dropped_attribute_count(msgpack_map_get_object(&body->via.map, kind)); +} + static uint64_t msgpack_object_hash(msgpack_object *object) { uint64_t hash; @@ -570,6 +596,8 @@ static int append_v1_logs_metadata_and_fields(struct opentelemetry_context *ctx, int trace_flags_set = FLB_FALSE; size_t attr_count = 0; struct flb_ra_value *ra_val; + msgpack_object *otlp_metadata; + msgpack_object *event_name; Opentelemetry__Proto__Common__V1__KeyValue **attrs = NULL; if (ctx == NULL || event == NULL || log_record == NULL) { @@ -824,6 +852,25 @@ static int append_v1_logs_metadata_and_fields(struct opentelemetry_context *ctx, } } + if (event->metadata && event->metadata->type == MSGPACK_OBJECT_MAP) { + otlp_metadata = msgpack_map_get_object(&event->metadata->via.map, "otlp"); + log_record->dropped_attributes_count = dropped_attribute_count(otlp_metadata); + + event_name = NULL; + if (otlp_metadata && otlp_metadata->type == MSGPACK_OBJECT_MAP) { + event_name = msgpack_map_get_object(&otlp_metadata->via.map, "event_name"); + } + + if (event_name && event_name->type == MSGPACK_OBJECT_STR) { + log_record->event_name = flb_strndup(event_name->via.str.ptr, + event_name->via.str.size); + if (!log_record->event_name) { + flb_errno(); + return -1; + } + } + } + return 0; } @@ -851,6 +898,9 @@ static void free_log_records(Opentelemetry__Proto__Logs__V1__LogRecord **logs, s if (log->severity_text != NULL && log->severity_text != protobuf_c_empty_string) { flb_free(log->severity_text); } + if (log->event_name != NULL && log->event_name != protobuf_c_empty_string) { + flb_free(log->event_name); + } if (log->span_id.data != NULL) { flb_free(log->span_id.data); } @@ -862,6 +912,36 @@ static void free_log_records(Opentelemetry__Proto__Logs__V1__LogRecord **logs, s } } +/* + * Release the log records of a request that was already sent, keeping its + * resources and scopes, so a later request of the same chunk does not repeat + * them. + */ +static void release_sent_log_records(Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *export_logs) +{ + size_t i; + size_t j; + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; + + for (i = 0; i < export_logs->n_resource_logs; i++) { + resource_log = export_logs->resource_logs[i]; + if (resource_log == NULL) { + continue; + } + + for (j = 0; j < resource_log->n_scope_logs; j++) { + scope_log = resource_log->scope_logs[j]; + if (scope_log == NULL || scope_log->log_records == NULL) { + continue; + } + + free_log_records(scope_log->log_records, scope_log->n_log_records); + scope_log->n_log_records = 0; + } + } +} + static void free_resource_logs(Opentelemetry__Proto__Logs__V1__ResourceLogs **resource_logs, size_t resource_count) { int i; @@ -1132,6 +1212,7 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, int ret; int record_type; int log_record_count; + int pending_records = 0; int max_scopes_limit; int max_resources; int native_otel = FLB_FALSE; @@ -1355,6 +1436,8 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, /* group body: $resource['attributes'] */ set_resource_attributes(ctx->ra_resource_attr, event.body, resource_log->resource); + resource_log->resource->dropped_attributes_count = + group_dropped_attribute_count(event.body, "resource"); /* group body: $schema_url */ set_resource_schema_url(ctx->ra_resource_schema_url, event.body, resource_log); @@ -1476,6 +1559,8 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, /* group body: $scope['attributes'] */ set_scope_attributes(ctx->ra_scope_attr, event.body, scope_log->scope); + scope_log->scope->dropped_attributes_count = + group_dropped_attribute_count(event.body, "scope"); /* group body: $scope['schema_url'] */ set_scope_schema_url(ctx->ra_scope_schema_url, event.body, scope_log); @@ -1582,13 +1667,14 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, ret = FLB_OK; log_record_count++; + pending_records++; scope_log->n_log_records = log_record_count; if (log_record_count >= ctx->batch_size) { ret = logs_flush_to_otel(ctx, event_chunk, &export_logs); - free_log_records(log_records, log_record_count); + release_sent_log_records(&export_logs); log_record_count = 0; - scope_log->n_log_records = 0; + pending_records = 0; /* Do not let a later batch overwrite this failure. */ if (ret != FLB_OK) { @@ -1599,7 +1685,11 @@ int otel_process_logs(struct flb_event_chunk *event_chunk, flb_log_event_decoder_destroy(decoder); - if (log_record_count > 0 && ret == FLB_OK) { + /* + * Records of earlier scopes are pending too: a chunk can end with a group + * that has no records yet, for example one opened by the previous append. + */ + if (pending_records > 0 && ret == FLB_OK) { ret = logs_flush_to_otel(ctx, event_chunk, &export_logs); } diff --git a/plugins/processor_content_modifier/cm.c b/plugins/processor_content_modifier/cm.c index 22180dbca17..ebc34831b2e 100644 --- a/plugins/processor_content_modifier/cm.c +++ b/plugins/processor_content_modifier/cm.c @@ -167,5 +167,5 @@ struct flb_processor_plugin processor_content_modifier_plugin = { .cb_process_traces = cb_process_traces, .cb_exit = cb_exit, .config_map = config_map, - .flags = 0 + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS }; diff --git a/plugins/processor_fts/CMakeLists.txt b/plugins/processor_fts/CMakeLists.txt new file mode 100644 index 00000000000..a27687dd11f --- /dev/null +++ b/plugins/processor_fts/CMakeLists.txt @@ -0,0 +1,28 @@ +# Embed the Lua helper sources as C string literals. +function(fts_embed_lua variable file) + file(READ "${CMAKE_CURRENT_SOURCE_DIR}/${file}" content) + string(REPLACE "\\" "\\\\" content "${content}") + string(REPLACE "\"" "\\\"" content "${content}") + string(REPLACE "\n" "\\n\"\n\"" content "${content}") + set(${variable} "${content}" PARENT_SCOPE) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${file}) +endfunction() + +fts_embed_lua(FTS_VALUE_HELPERS fts_value_helpers.lua) +fts_embed_lua(FTS_LOG_HELPERS fts_log_helpers.lua) +fts_embed_lua(FTS_TELEMETRY_HELPERS fts_telemetry_helpers.lua) +configure_file(fts_helpers.h.in fts_helpers.h @ONLY) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +set(src + fts.c + fts_runtime.c + fts_values.c + fts_logs.c + fts_proto.c + fts_telemetry.c + fts_metrics.c + fts_traces.c + ) + +FLB_PLUGIN(processor_fts "${src}" "") diff --git a/plugins/processor_fts/README.md b/plugins/processor_fts/README.md new file mode 100644 index 00000000000..33429afc313 --- /dev/null +++ b/plugins/processor_fts/README.md @@ -0,0 +1,340 @@ +# Fluent Typed Scripting (FTS) processor + +FTS is a processor for writing telemetry policies in Lua. Policies get typed +access to the record and its context: integer widths, binary values, and +nanosecond timestamps are preserved, and OpenTelemetry resources, scopes, and +attributes are exposed as properties instead of raw metadata. + +FTS supports logs, metrics, and traces. Its configuration name is `fts`. + +| Signal | Callback | Called once per | +| ------- | ----------------- | --------------- | +| Logs | `process(log)` | log record | +| Metrics | `process(metric)` | metric family | +| Traces | `process(span)` | span | + +## Quick start + +```yaml +pipeline: + inputs: + - name: opentelemetry + port: 4318 + processors: + logs: + - name: fts + script: policy.lua + outputs: + - name: stdout + match: '*' + format: otlp_json +``` + +```lua +local fb = require("fluentbit") + +function process(log) + if log.body:get_string("path") == "/health" then + return fb.DROP + end + + fb.redact(log.body, {"authorization", "password", "access_token"}) + + if fb.is_otel(log) then + log.resource.attributes:set("deployment.environment.name", "production") + log.attributes:set("sanitized", true) + end +end +``` + +Use one processor entry and one script for each signal. + +## Configuration + +| Key | Default | Description | +| ------------------- | --------- | ----------- | +| `script` | | Path to the Lua policy file. Set either `script` or `code`. | +| `code` | | Inline Lua policy. Set either `script` or `code`. | +| `call` | `process` | Name of the callback function. | +| `error_mode` | `drop` | What happens to an item when its policy fails: `drop` removes it, `keep` forwards the original. | +| `instruction_limit` | `1000000` | Maximum Lua instructions per callback and during script load. Minimum `1000`. `0` is allowed only with `jit: true`. | +| `jit` | `false` | Enable the LuaJIT compiler. Requires `instruction_limit: 0`. | +| `memory_limit` | `32M` | Maximum memory for the policy: the Lua heap plus the typed values it holds. Minimum `262144`. | +| `output_limit` | `64M` | Maximum serialized size of one processed batch. Minimum `65536`. | + +Size suffixes are decimal: `64K` is 64000 bytes. + +## Callback results + +| Return value | Effect | +| -------------------- | ------ | +| `nil` or `fb.KEEP` | Commit all changes made by the callback. | +| `fb.DROP` | Remove the item. | +| Lua error | Roll back every change made by the callback, then apply `error_mode`. | + +Changes are transactional per item. Edits to the body, metadata, timestamp, +resource, and scope are committed or rolled back together. + +For logs, if a processed batch exceeds `output_limit`, the whole batch is +treated as failed: `drop` discards it and `keep` forwards the original batch. +For metrics and traces the limit is checked per item: an item whose result does +not fit follows `error_mode` (`drop` removes it, `keep` forwards the original +item), and the whole batch fails only when the kept originals do not fit +either. Internal failures, such as a batch that cannot be decoded, apply +`error_mode` to the whole batch. + +A batch is the set of records handed to one callback run. Small chunks are +processed as one batch. Larger buffers, such as complete chunks reaching output +processors, are processed in batches of about 64 KiB or 1024 records, so +decoded records never hold more than one batch in memory. Groups, such as one +OpenTelemetry resource and scope, are never split across batches. + +A group can also span appends, for example when its start and end markers +arrive in separate forward messages. A group still open at the end of a batch +stays open for the records that follow. Records that arrive before the end +marker of a group opened earlier are processed without a context, keep their +place in that group, and cannot set one. + +For privacy policies such as redaction, keep the default `error_mode: drop`. +With `keep`, an item whose policy fails is forwarded unmodified. + +## Typed values + +Bodies, attributes, and other structured fields are typed maps and arrays. + +### Map and array methods + +| Method | Applies to | Description | +| -------------------------- | ------------ | ----------- | +| `v:get(key)` | map, array | Child value, or `nil` when missing. Array indexes start at 1. | +| `v:get_string(key)` | map | String value, or `nil` when missing. Fails on another type. | +| `v:get_number(key)` | map | Numeric value as a Lua number, or `nil` when missing. | +| `v:get_bool(key)` | map | Boolean value, or `nil` when missing. | +| `v:set(key, value)` | map, array | Set a value. Replaces every entry with the same key. | +| `v:remove(key)` | map | Remove every entry with that key. | +| `v:ensure_map(key)` | map | Return the child map, creating it when missing. | +| `v:append(value)` | array | Append a value. | +| `v:keys()` | map | Lua table with the keys in order. | +| `v:len()` | map, array | Number of entries. | +| `v:type()` | any | `map`, `array`, `string`, `bytes`, `bool`, `int64`, `uint64`, `double`, or `null`. | +| `v:value()` | scalar | Lua value. 64-bit integers are returned as exact decimal strings. | +| `v:number()` | numeric | Lua number. Fails when an integer is beyond ±(2^53 − 1). | +| `v:copy()` | any | Independent, writable copy. | + +Keys match exact bytes; a dot in a key has no special meaning. + +### Constructors + +| Constructor | Creates | +| ------------------ | ------- | +| `fb.map()` | Empty map | +| `fb.array()` | Empty array | +| `fb.int64(v)` | Signed 64-bit integer from a decimal string or a safe Lua integer | +| `fb.uint64(v)` | Unsigned 64-bit integer from a decimal string or a safe Lua integer | +| `fb.bytes(s)` | Binary value | +| `fb.null` | Null value | + +Lua strings, booleans, and numbers can be passed to `set` and `append` +directly. Lua numbers are stored as doubles; use `fb.int64` or `fb.uint64` for +integers. Lua tables and `nil` are not values: use `fb.map()`, `fb.array()`, +and `fb.null`. + +### Value lifetime + +Log values passed to a callback, and the views obtained from them, are valid +only during that callback. Using one later raises an error. Metric and span +values are snapshots made for the callback: they stay readable afterwards, but +changing them never affects later items or committed output, and keeping them +retains their memory. To keep data between callbacks, store `v:copy()`. Copies +are independent: changing them never changes a committed record. + +Lua globals persist for the lifetime of the processor instance. + +## Logs + +| Property | Description | +| ------------------------------------------- | ----------- | +| `log.body` | Record body map. Can be replaced with another map. | +| `log.metadata` | Record metadata map. | +| `log.timestamp` | `{sec, nsec}` table. Assign a new table to change the event time. | +| `log.tag` | Routing tag, read-only. | +| `log.attributes` | OpenTelemetry log attributes. | +| `log.resource.attributes`, `.schema_url` | OpenTelemetry resource. | +| `log.scope.attributes`, `.name`, `.version`, `.schema_url` | OpenTelemetry instrumentation scope. | +| `log.context.attributes`, `.metadata` | Group attributes and metadata of non-OpenTelemetry grouped logs. | + +OpenTelemetry properties exist only for logs that carry OpenTelemetry context, +for example logs received by the `opentelemetry` input. Use `fb.is_otel(log)` +to check. To give ordinary logs OpenTelemetry context, place the +`opentelemetry_envelope` processor before FTS. + +Resource, scope, and context changes apply only to the current record. Record +order is preserved. + +### Log helpers + +| Helper | Description | +| ------ | ----------- | +| `fb.is_otel(log)` | `true` when the log has OpenTelemetry context. | +| `fb.set_severity(log, level)` | Set severity text and number from a level name: `trace`, `debug`, `info`, `warn`, `error`, `fatal`, and common aliases. | +| `fb.set_trace_context(log, trace_hex, span_hex)` | Set the trace and span IDs from hexadecimal strings. IDs must be nonzero. | +| `fb.log_field(log, name)` | Read an OpenTelemetry log field such as `severity_number`, `trace_id`, or `observed_timestamp`. | + +### Example: parse an embedded JSON message + +```lua +local fb = require("fluentbit") + +function process(log) + local text = log.body:get_string("message") + if not text or not text:match("^%s*{") then + return + end + + local ok, app = pcall(fb.parse_json, text) + if not ok then + return fb.DROP + end + + fb.redact(app, {"authorization", "password"}) + log.body = app + + if fb.is_otel(log) then + local level = app:get_string("level") + if level then + fb.set_severity(log, level) + end + + local time = app:get_string("time") + if time then + log.timestamp = fb.parse_time(time) + end + end +end +``` + +## Metrics + +`process(metric)` runs once per metric family. Resource and scope changes apply +to that family only. + +```lua +local fb = require("fluentbit") + +function process(metric) + if metric.name == "http.server.duration" then + metric.name = "http.server.request.duration" + fb.scale_metric(metric, 0.001, "s") + end + + for i = 1, metric.datapoints:len() do + metric.datapoints:get(i):get("attributes"):remove("request.id") + end +end +``` + +| Property | Description | +| --------------------------------------------- | ----------- | +| `metric.name`, `.description`, `.unit` | Family identity. | +| `metric.type` | `gauge`, `sum`, `histogram`, `exponential_histogram`, or `summary`. Only gauge and sum can be converted into each other. | +| `metric.datapoints` | Array of datapoint maps. | +| `metric.aggregation_temporality` | `unspecified`, `delta`, or `cumulative`. | +| `metric.is_monotonic` | Monotonicity of a sum. | +| `metric.resource`, `metric.scope` | Same properties as for logs. | + +Datapoint maps use OpenTelemetry field names in snake case, for example +`attributes`, `time_unix_nano`, `start_time_unix_nano`, `as_int`, `as_double`, +`count`, `sum`, `bucket_counts`, `explicit_bounds`, `quantile_values`, and +`exemplars`. Timestamps and counts are exact integers. Histogram buckets are +per-bucket counts. + +Removing a datapoint attribute can make two series identical, which fails the +policy. Use `fb.aggregate_metric` to combine them deliberately. + +### Metric helpers + +| Helper | Description | +| ------ | ----------- | +| `fb.metric_value(point)` | Scalar value of a gauge or sum datapoint. | +| `fb.set_metric_value(point, value)` | Set the scalar value of a datapoint. | +| `fb.scale_metric(metric, factor, unit)` | Multiply values by a positive factor and optionally set the unit. Applies to gauges, sums, histograms, and summaries, including bounds and exemplars. | +| `fb.aggregate_metric(metric, keys, op)` | Keep only the listed attributes and combine equal series. `op` is `sum`, or for gauges also `min`, `max`, `mean`, or `count`. | +| `fb.merge_histogram_buckets(metric, bounds)` | Merge histogram buckets into a subset of the existing bounds. | +| `fb.copy_metric(metric, name, description, unit)` | Emit a copy of the family with a new name. | +| `fb.extract_metric(metric, field, name, monotonic)` | Emit a sum derived from a histogram or summary: `count`, `sum`, or `avg`. | + +Emitted families follow the original and continue through the pipeline. They +are not passed to the callback again. Returning `fb.DROP` removes the original +and keeps the emitted families; an error discards both. + +Scalar values use the OpenTelemetry int64 and double types. Native unsigned +64-bit values up to 2^63 − 1 are passed to the policy as int64. Families with +larger values are not passed to the policy; `error_mode` decides whether they +are kept unchanged or dropped. + +## Traces + +`process(span)` runs once per span. Resource and scope changes apply to that +span only. + +```lua +local fb = require("fluentbit") + +function process(span) + if span.attributes:get_string("http.route") == "/health" then + return fb.DROP + end + + fb.redact(span.attributes, {"authorization", "password"}) + fb.redact(span.events, {"authorization", "password"}) +end +``` + +| Property | Description | +| -------- | ----------- | +| `span.name`, `.trace_state`, `.kind`, `.flags` | Span fields. `kind` is the numeric OpenTelemetry span kind. | +| `span.trace_id`, `.span_id`, `.parent_span_id` | Hexadecimal IDs. An empty parent ID marks a root span. | +| `span.start_time_unix_nano`, `.end_time_unix_nano` | Exact nanosecond timestamps. The end must not precede the start. | +| `span.attributes` | Attribute map. | +| `span.status` | Map with `code` and `message`. | +| `span.events`, `span.links` | Arrays of event and link maps. | +| `span.dropped_attributes_count`, `.dropped_events_count`, `.dropped_links_count` | Dropped counts. | +| `span.resource`, `span.scope` | Same properties as for logs. | + +Dropping spans or changing IDs can break trace topology; FTS does not repair +parent references. + +## Common helpers + +| Helper | Description | +| ------ | ----------- | +| `fb.redact(value, keys, replacement)` | Replace the values of matching keys in nested maps and arrays. Key matching is case-insensitive. The default replacement is `[REDACTED]`. | +| `fb.keep_keys(map, keys)` | Remove every key not listed. | +| `fb.parse_json(text)` | Parse a JSON document into typed values. Integer widths are preserved; duplicate keys and nesting beyond 64 levels are rejected. Input is limited to 1 MiB. | +| `fb.parse_time(text)` | Parse an RFC 3339 timestamp into `{sec, nsec}`. | + +Key-based redaction does not inspect free-form strings; secrets embedded in +message text need an explicit string policy. + +## Limits and sandbox + +- Values can have at most 64 levels of nesting. +- OpenTelemetry strings and keys must be valid UTF-8 without NUL characters; + use `fb.bytes` for binary data. +- Only `require("fluentbit")` is available. The filesystem, process, FFI, + debug, coroutine, and module loading libraries are disabled, as are `print`, + `xpcall`, `getfenv`, `setfenv`, and `module`. `collectgarbage` accepts only + `collect`, `count`, and `step`. +- The instruction limit counts Lua instructions, not elapsed time. Lua string + pattern functions run in C and are not interrupted; avoid unbounded patterns + on untrusted input. +- With `jit: true` there is no CPU limit. Use it only for policies whose + execution time you control. +- The sandbox is not a security boundary: policies are trusted configuration. +- Each processor instance has its own Lua state. + +## Diagnostics + +Each batch logs at most five policy errors plus a summary count. The processor +error metric counts batches with handled policy failures, and the drop metrics +count removed records. diff --git a/plugins/processor_fts/fts.c b/plugins/processor_fts/fts.c new file mode 100644 index 00000000000..fe62b3a5bf9 --- /dev/null +++ b/plugins/processor_fts/fts.c @@ -0,0 +1,197 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#include "fts.h" +#include "fts_telemetry.h" + +static int cb_exit(struct flb_processor_instance *ins, void *data) +{ + struct fts_runtime *ctx = data; + + if (!ctx) { + return FLB_PROCESSOR_SUCCESS; + } + + fts_runtime_destroy(ctx); + flb_free(ctx); + + return FLB_PROCESSOR_SUCCESS; +} + +static int config_validate(struct flb_processor_instance *ins, struct fts_runtime *ctx) +{ + if ((!ctx->script && !ctx->code) || (ctx->script && ctx->code)) { + flb_plg_error(ins, "FTS requires exactly one of script or code"); + return -1; + } + + if (strcmp(ctx->error_mode, "keep") != 0 && strcmp(ctx->error_mode, "drop") != 0) { + flb_plg_error(ins, "error_mode must be keep or drop"); + return -1; + } + + if (ctx->jit && ctx->instruction_limit != 0) { + flb_plg_error(ins, "jit requires instruction_limit=0"); + return -1; + } + + if (!ctx->jit && ctx->instruction_limit < 1000) { + flb_plg_error(ins, "instruction_limit must be between 1000 and %d with jit disabled", + INT_MAX); + return -1; + } + + if (ctx->memory_limit < 262144) { + flb_plg_error(ins, "memory_limit must be at least 262144 bytes (256 KiB)"); + return -1; + } + + if (ctx->output_limit < 65536) { + flb_plg_error(ins, "output_limit must be at least 65536 bytes (64 KiB)"); + return -1; + } + + return 0; +} + +static int cb_init(struct flb_processor_instance *ins, void *source, int type, + struct flb_config *config) +{ + int ret; + lua_State *l; + struct fts_runtime *ctx; + + if (ins->event_type != FLB_PROCESSOR_LOGS && ins->event_type != FLB_PROCESSOR_METRICS && + ins->event_type != FLB_PROCESSOR_TRACES) { + flb_plg_error(ins, "FTS supports logs, metrics and traces"); + return FLB_PROCESSOR_FAILURE; + } + + ctx = flb_calloc(1, sizeof(struct fts_runtime)); + if (!ctx) { + flb_errno(); + return FLB_PROCESSOR_FAILURE; + } + + if (ins->event_type == FLB_PROCESSOR_LOGS) { + ctx->adapter = &fts_logs_adapter; + } + else if (ins->event_type == FLB_PROCESSOR_METRICS) { + ctx->adapter = &fts_metrics_adapter; + } + else { + ctx->adapter = &fts_traces_adapter; + } + + ret = flb_processor_instance_config_map_set(ins, ctx); + if (ret != 0) { + flb_plg_error(ins, "invalid FTS configuration value"); + cb_exit(ins, ctx); + return FLB_PROCESSOR_FAILURE; + } + + if (config_validate(ins, ctx) != 0) { + cb_exit(ins, ctx); + return FLB_PROCESSOR_FAILURE; + } + + /* Load the helpers and the policy script into a private Lua state */ + ret = fts_runtime_initialize(ctx, config); + if (ret != FLB_PROCESSOR_SUCCESS) { + l = ctx->vm ? ctx->vm->state : NULL; + if (!l) { + flb_plg_error(ins, "FTS initialization failed: VM allocation error"); + } + else if (lua_type(l, -1) == LUA_TSTRING) { + flb_plg_error(ins, "FTS initialization failed: %.256s", lua_tostring(l, -1)); + } + else { + flb_plg_error(ins, "FTS initialization failed: non-string error"); + } + cb_exit(ins, ctx); + return FLB_PROCESSOR_FAILURE; + } + + flb_processor_instance_set_context(ins, ctx); + + return FLB_PROCESSOR_SUCCESS; +} + +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_STR, "script", NULL, + 0, FLB_TRUE, offsetof(struct fts_runtime, script), + "Path to the Lua policy script. Exclusive with 'code'." + }, + { + FLB_CONFIG_MAP_STR, "code", NULL, + 0, FLB_TRUE, offsetof(struct fts_runtime, code), + "Inline Lua policy script. Exclusive with 'script'." + }, + { + FLB_CONFIG_MAP_STR, "call", "process", + 0, FLB_TRUE, offsetof(struct fts_runtime, call), + "Name of the Lua function invoked for every telemetry item." + }, + { + FLB_CONFIG_MAP_STR, "error_mode", "drop", + 0, FLB_TRUE, offsetof(struct fts_runtime, error_mode), + "Action when a policy fails: 'drop' the item or 'keep' the original." + }, + { + FLB_CONFIG_MAP_BOOL, "jit", "false", + 0, FLB_TRUE, offsetof(struct fts_runtime, jit), + "Enable the LuaJIT compiler for trusted policies. Requires instruction_limit=0." + }, + { + FLB_CONFIG_MAP_INT, "instruction_limit", "1000000", + 0, FLB_TRUE, offsetof(struct fts_runtime, instruction_limit), + "Maximum number of Lua instructions per callback." + }, + { + FLB_CONFIG_MAP_SIZE, "output_limit", "64M", + 0, FLB_TRUE, offsetof(struct fts_runtime, output_limit), + "Maximum serialized size of a processed batch." + }, + { + FLB_CONFIG_MAP_SIZE, "memory_limit", "32M", + 0, FLB_TRUE, offsetof(struct fts_runtime, memory_limit), + "Maximum memory used by the policy: Lua heap plus typed values." + }, + + /* EOF */ + {0} +}; + +struct flb_processor_plugin processor_fts_plugin = { + .name = "fts", + .description = "Fluent Typed Scripting: Lua-powered typed telemetry policies", + .cb_init = cb_init, + .cb_process_logs = fts_process_logs, + .cb_process_metrics = fts_process_metrics, + .cb_process_traces = fts_process_traces, + .cb_exit = cb_exit, + .config_map = config_map, + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS +}; diff --git a/plugins/processor_fts/fts.h b/plugins/processor_fts/fts.h new file mode 100644 index 00000000000..86cc2d7deae --- /dev/null +++ b/plugins/processor_fts/fts.h @@ -0,0 +1,154 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_PROCESSOR_FTS_H +#define FLB_PROCESSOR_FTS_H + +#include +#include +#include + +#define FTS_MAX_DEPTH 64 + +/* Each signal owns its representation, validation and commit semantics. */ +struct fts_signal_adapter { + int event_type; + int (*register_helpers)(lua_State *l); + lua_CFunction run_callback; +}; + +enum fts_key { + FTS_KEY_TAG, FTS_KEY_BODY, FTS_KEY_METADATA, FTS_KEY_TIMESTAMP, + FTS_KEY_SEC, FTS_KEY_NSEC, FTS_KEY_CONTEXT, FTS_KEY_ATTRIBUTES, + FTS_KEY_COUNT +}; + +struct fts_runtime; + +/* Reversible in-place edits of chunk-owned log values. */ +struct fts_undo_entry { + int type; + struct cfl_list *list; + struct cfl_kvpair *pair; + struct cfl_list *prev; + struct cfl_array *array; + size_t index; + struct cfl_variant *old; +}; + +struct fts_value { + struct cfl_variant *owned; + unsigned int readonly : 1; + unsigned int dirty : 1; + unsigned int depth_changed : 1; + unsigned int borrowed : 1; + unsigned int record : 1; + unsigned int inplace : 1; + struct fts_runtime *ctx; + uint64_t generation; + size_t charged; + + /* + * Views: owning root and the node resolved at the root's last epoch. + * Roots: epoch advances on every write, invalidating view caches. + */ + struct fts_value *root; + struct cfl_variant *cache; + uint64_t epoch; +}; + +struct fts_runtime { + struct flb_luajit *vm; + char *script; + char *code; + char *call; + char *error_mode; + const struct fts_signal_adapter *adapter; + const void *value_metatable; + int value_metatable_ref; + const void *empty_metatable; + int empty_metatable_ref; + int key_refs[FTS_KEY_COUNT]; + int tag_ref; + uint64_t generation; + struct fts_undo_entry *undo; + size_t undo_count; + size_t undo_size; + size_t undo_held; + size_t undo_charged; + int inplace_ok; + int tag_ready; + int callback_ref; + int policy_ref; + int module_ref; + int helpers_env_ref; + int jit; + int instruction_limit; + int instructions_left; + size_t memory_limit; + size_t lua_bytes; + size_t native_bytes; + size_t native_allocated; /* native bytes charged since the last GC pacing */ + size_t output_limit; + lua_Alloc original_allocator; + void *allocator_data; +}; + +extern const struct fts_signal_adapter fts_logs_adapter; + +/* runtime: Lua VM lifecycle, policy calls and error reporting */ +int fts_runtime_initialize(struct fts_runtime *ctx, struct flb_config *config); +void fts_runtime_destroy(struct fts_runtime *ctx); +void fts_report_policy_error(struct flb_processor_instance *ins); +void fts_sanitizer_forget_unwound_frames(void); +int fts_runtime_call(struct fts_runtime *ctx, void *data, int results); +int fts_register_values(lua_State *l); +void fts_collect(struct fts_runtime *ctx, int emergency); + +/* cached field-name keys */ +void fts_getfield(lua_State *l, int index, enum fts_key key); +void fts_setfield(lua_State *l, int index, enum fts_key key); + +/* Lua values backed by msgpack or cfl_variant data */ +struct fts_value *fts_push_empty_map(lua_State *l); +struct fts_value *fts_push_borrowed(lua_State *l, struct cfl_variant *source); + +/* undo log for in-place edits */ +void fts_undo_rollback(struct fts_runtime *ctx, size_t mark); +void fts_undo_commit(struct fts_runtime *ctx); + +/* helpers and value utilities */ +int fts_load_helpers(lua_State *l, const char *code, size_t size, const char *name); +int fts_values_equal(struct cfl_variant *a, struct cfl_variant *b, int depth); +int fts_valid_utf8(const char *text, size_t length); +struct cfl_variant *fts_copy_value(struct cfl_variant *value, int depth); +struct cfl_variant *fts_copy_value_in(struct cfl_variant *value, int depth, + struct cfl_arena *arena); +size_t fts_value_cost(struct cfl_variant *value, int depth); +void fts_charge_value(lua_State *l, struct fts_value *owner, size_t bytes); +struct cfl_kvpair *fts_find_pair(struct cfl_variant *value, const char *key, size_t length); +struct cfl_variant *fts_resolve_value(lua_State *l, int index, int depth); +struct fts_value *fts_push_owned(lua_State *l, struct cfl_variant *value, int readonly); + +/* logs signal */ + +int fts_process_logs(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len); + +#endif diff --git a/plugins/processor_fts/fts_helpers.h.in b/plugins/processor_fts/fts_helpers.h.in new file mode 100644 index 00000000000..43c7d4afdff --- /dev/null +++ b/plugins/processor_fts/fts_helpers.h.in @@ -0,0 +1,15 @@ +/* Generated from the processor_fts Lua helper sources. Do not edit. */ + +#ifndef FLB_PROCESSOR_FTS_HELPERS_H +#define FLB_PROCESSOR_FTS_HELPERS_H + +static const char fts_value_helpers[] = +"@FTS_VALUE_HELPERS@"; + +static const char fts_log_helpers[] = +"@FTS_LOG_HELPERS@"; + +static const char fts_telemetry_helpers[] = +"@FTS_TELEMETRY_HELPERS@"; + +#endif diff --git a/plugins/processor_fts/fts_log_helpers.lua b/plugins/processor_fts/fts_log_helpers.lua new file mode 100644 index 00000000000..f19f9d7a39c --- /dev/null +++ b/plugins/processor_fts/fts_log_helpers.lua @@ -0,0 +1,171 @@ +local fb = package.loaded.fluentbit +function fb.is_otel(log) + local schema = rawget(log, 'context') and rawget(log, 'context').metadata:get('schema') + return schema and schema:type() == 'string' and schema:value() == 'otlp' or false +end +local function read_map(map, key, created) + local value = map:get(key) + if value then return value end + value = map:ensure_map(key) + created[#created+1] = {map, key} + return value +end +local function semantic(map, created) + return setmetatable({}, { + __index = function(_, key) + if key == 'attributes' then return read_map(map, key, created) end + local value = map:get(key); if value then return value:value() end + end, + __newindex = function(_, key, value) + assert(key == 'name' or key == 'version' or key == 'schema_url', 'unsupported context property') + assert(type(value) == 'string', 'context property must be a string') + map:set(key, value) + end, + __metatable = 'telemetry context' + }) +end +function fb._attach(log) + local created + local context = rawget(log, 'context') + local root_metadata = rawget(log, 'metadata') + local otel = fb.is_otel(log) + local function identity(name) + -- Missing IDs are regenerated when the group is rebuilt. + local value = rawget(context, 'metadata'):get(name) + if value == nil then return nil end + assert(value:type() == 'int64' or value:type() == 'uint64', 'invalid transport ID') + return value:value() + end + local resource_id, scope_id + if otel then + created = {} + resource_id = identity('resource_id') + scope_id = identity('scope_id') + log.resource = semantic(read_map(rawget(context, 'attributes'), 'resource', created), created) + log.scope = semantic(read_map(rawget(context, 'attributes'), 'scope', created), created) + log.attributes = read_map(read_map(rawget(log, 'metadata'), 'otlp', created), 'attributes', created) + end + local resource, scope, attrs = rawget(log, 'resource'), rawget(log, 'scope'), rawget(log, 'attributes') + local original_time = otel and fb.log_field(log, 'timestamp') + original_time = original_time and original_time:value() + local function validate(sec, nsec) + assert(rawget(log, 'context') == context, 'context root cannot be replaced') + assert(rawget(log, 'metadata') == root_metadata, 'metadata root cannot be replaced') + assert(fb.is_otel(log) == otel, 'transport schema cannot be changed') + if otel then + assert(rawget(log, 'resource') == resource and rawget(log, 'scope') == scope and rawget(log, 'attributes') == attrs, + 'semantic aliases cannot be replaced') + assert(rawget(context, 'metadata'):get_string('schema') == 'otlp' + and identity('resource_id') == resource_id + and identity('scope_id') == scope_id, + 'transport identifiers cannot be changed') + if rawget(log, 'timestamp').sec == sec and rawget(log, 'timestamp').nsec == nsec then + local current = fb.log_field(log, 'timestamp') + assert((current and current:value()) == original_time, 'change event time through log.timestamp') + end + end + end + return validate,created +end +function fb._cleanup(created) + for i=#created,1,-1 do + local map,key = created[i][1],created[i][2] + -- A policy may have removed an ancestor through the advanced context API. + local ok,value = pcall(function() return map:get(key) end) + if ok and value and value:type() == 'map' and value:len() == 0 then + map:remove(key) + end + end +end +function fb.attributes(log) + assert(fb.is_otel(log), 'OTel log required') + return rawget(log, 'attributes') +end +function fb.log_field(log, name) + assert(fb.is_otel(log), 'OTel log required') + local otlp = rawget(log, 'metadata'):get('otlp') + return (otlp and otlp:get(name)) or rawget(log, 'metadata'):get(name) +end +function fb.set_severity(log, level) + assert(fb.is_otel(log), 'OTel log required') + local names = {trace=1, debug=5, info=9, information=9, informational=9, + warn=13, warning=13, error=17, fatal=21, critical=21} + assert(type(level) == 'string', 'severity must be a named level') + local number = names[level:lower()]; assert(number, 'unknown severity level') + local otlp = rawget(log, 'metadata'):ensure_map('otlp') + otlp:set('severity_text', level); otlp:set('severity_number', fb.int64(number)) +end +function fb.set_trace_context(log, trace, span) + assert(fb.is_otel(log), 'OTel log required') + local function decode(s, length) + assert(type(s) == 'string' and #s == length * 2 and not s:find('[^%x]') + and s:find('[1-9a-fA-F]'), 'invalid correlation ID') + return fb.bytes((s:gsub('..', function(pair) return string.char(tonumber(pair, 16)) end))) + end + local trace_value = decode(trace, 16); local span_value = decode(span, 8) + local otlp = rawget(log, 'metadata'):ensure_map('otlp') + otlp:set('trace_id', trace_value); otlp:set('span_id', span_value) +end +function fb._validate(log, original_sec, original_nsec, changed) + if not fb.is_otel(log) then return end + for _,kind in ipairs({'resource','scope'}) do + local map = rawget(log, 'context').attributes:get(kind) + if map then + assert(map:type() == 'map', kind .. ' must be a map') + for _,name in ipairs({'schema_url','name','version'}) do + local value = map:get(name) + assert(not changed[kind .. '.' .. name] or not value or value:type() == 'string', kind .. '.' .. name .. ' must be a string') + end + local attrs = map:get('attributes') + if attrs and changed[kind .. '.attributes'] then + assert(attrs:type() == 'map', kind .. '.attributes must be a map') + local seen = {} + for _,key in ipairs(attrs:keys()) do + assert(not seen[key], 'duplicate attribute key'); seen[key] = true + end + end + local count = map:get('dropped_attributes_count') + if count and changed[kind .. '.dropped_attributes_count'] then + assert(count:type() == 'int64' or count:type() == 'uint64', 'dropped count must be an integer') + assert(count:number() >= 0 and count:number() <= 4294967295, 'dropped count out of range') + end + end + end + assert(rawget(log, 'attributes'):type() == 'map', 'log.attributes must be a map') + local seen = {} + for _,key in ipairs(changed.attributes and rawget(log, 'attributes'):keys() or {}) do + assert(not seen[key], 'duplicate attribute key'); seen[key] = true + end + local otlp = rawget(log, 'metadata'):get('otlp') + for _,name in ipairs({'severity_number','trace_flags','dropped_attributes_count'}) do + local value = fb.log_field(log, name) + if value and changed[name] then + assert(value:type() == 'int64' or value:type() == 'uint64', name .. ' must be an integer') + local number = value:number() + assert(number>=0 and number <= (name == 'severity_number' and 24 or (name == 'trace_flags' and 255 or 4294967295)), name .. ' out of range') + end + end + for _,name in ipairs({'trace_id','span_id'}) do + local value = fb.log_field(log, name) + if value and changed[name] then + local str = value:value() + assert(value:type() == 'bytes' and #str == (name == 'trace_id' and 16 or 8) + and not str:match('^%z+$'), 'invalid ' .. name) + end + end + local name = fb.log_field(log, 'event_name') + assert(not changed.event_name or not name or name:type() == 'string', 'event_name must be a string') + local severity = fb.log_field(log, 'severity_text') + assert(not changed.severity_text or not severity or severity:type() == 'string', 'severity_text must be a string') + for _,name in ipairs({'timestamp','observed_timestamp'}) do + local value = fb.log_field(log, name) + if value and changed[name] then + assert(value:type() == 'uint64' or (value:type() == 'int64' and value:value():sub(1,1) ~= '-'), + name .. ' must be a nonnegative integer') + end + end + if rawget(log, 'timestamp').sec ~= original_sec or rawget(log, 'timestamp').nsec ~= original_nsec then + local str = string.format('%.0f', rawget(log, 'timestamp').sec) .. string.format('%09d',rawget(log, 'timestamp').nsec) + otlp:set('timestamp', fb.uint64(str)) + end +end diff --git a/plugins/processor_fts/fts_logs.c b/plugins/processor_fts/fts_logs.c new file mode 100644 index 00000000000..62fbb7b4c3f --- /dev/null +++ b/plugins/processor_fts/fts_logs.c @@ -0,0 +1,1855 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "fts.h" +#include "fts_helpers.h" + +/* Log semantics remain in the log adapter. */ +static int values_equal(struct cfl_variant *a, struct cfl_variant *b); +static struct cfl_object *object_from_variant(struct cfl_variant *v); + +/* All Lua allocation, callbacks and validation run within one protected call. */ +struct record_call { + struct fts_runtime *ctx; + struct flb_mp_chunk_cobj *chunk; + struct cfl_variant *result_body; + struct cfl_variant *result_metadata; + struct flb_mp_chunk_record *record; + struct cfl_object *attributes; + struct cfl_object *metadata; + const char *tag; + int tag_len; +}; + +static struct fts_value *push_clone_in(lua_State *l, struct cfl_variant *source, + struct flb_mp_chunk_cobj *chunk) +{ + struct fts_value *value; + struct cfl_arena *arena; + size_t cost; + + value = fts_push_owned(l, NULL, FLB_FALSE); + cost = fts_value_cost(source, 0); + fts_charge_value(l, value, cost); + + arena = flb_mp_chunk_cobj_value_pool_get(chunk, cost > SIZE_MAX / 4 ? SIZE_MAX : cost * 4); + value->owned = fts_copy_value_in(source, 0, arena); + if (!value->owned && arena) { + value->owned = fts_copy_value(source, 0); + } + + if (!value->owned) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + + return value; +} + +static struct fts_value *push_clone(lua_State *l, struct cfl_variant *source) +{ + return push_clone_in(l, source, NULL); +} + +static int has_otel_context(struct cfl_object *obj) +{ + struct cfl_kvpair *pair; + + if (!obj || !obj->variant || obj->variant->type != CFL_VARIANT_KVLIST) { + return FLB_FALSE; + } + + pair = fts_find_pair(obj->variant, "schema", 6); + return pair && pair->val->type == CFL_VARIANT_STRING && + cfl_variant_size_get(pair->val) == 4 && + memcmp(pair->val->data.as_string, "otlp", 4) == 0; +} + +static struct cfl_variant *raw_field(lua_State *l, int table, enum fts_key key) +{ + struct cfl_variant *v; + + fts_getfield(l, table, key); + v = fts_resolve_value(l, -1, 0); + lua_pop(l, 1); + + return v; +} + +/* + * Push the field to pin its owner; return its tree only when the policy changed it. + * The caller copies it into the chunk after the protected call. + */ +static struct cfl_variant *push_record_result(lua_State *l, int table, enum fts_key key, + struct fts_value *original) +{ + struct cfl_variant *value; + + fts_getfield(l, table, key); + if (lua_touserdata(l, -1) == original && (!original->dirty || original->inplace)) { + /* In-place edits already live in the chunk's tree. */ + if (original->inplace && original->depth_changed && + fts_value_cost(original->owned, 0) == SIZE_MAX) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + return NULL; + } + + value = fts_resolve_value(l, -1, 0); + + /* Keep nesting failures item-local instead of failing the chunk copy. */ + if ((lua_touserdata(l, -1) != original || original->depth_changed) && + fts_value_cost(value, 0) == SIZE_MAX) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + + return value; +} + +/* Lua may retain policy trees, so the chunk receives its own copy. */ +static struct cfl_object *result_object(struct flb_mp_chunk_cobj *chunk, + struct cfl_variant *source) +{ + struct cfl_variant *value; + struct cfl_object *obj; + struct cfl_arena *arena; + size_t cost; + + cost = fts_value_cost(source, 0); + if (cost == SIZE_MAX) { + return NULL; + } + + arena = flb_mp_chunk_cobj_value_pool_get(chunk, cost > SIZE_MAX / 4 ? SIZE_MAX : cost * 4); + value = fts_copy_value_in(source, 0, arena); + if (!value && arena) { + value = fts_copy_value(source, 0); + } + + if (!value) { + return NULL; + } + + obj = object_from_variant(value); + if (!obj) { + cfl_variant_destroy(value); + } + + return obj; +} + +static void push_field_clone(lua_State *l, int table, enum fts_key key) +{ + struct cfl_variant *v; + + v = raw_field(l, table, key); + push_clone(l, v); +} + +static void validate_map(lua_State *l, struct cfl_variant *v, const char *name) +{ + if (v == NULL || v->type != CFL_VARIANT_KVLIST) { + luaL_error(l, "%s must be a typed map", name); + } +} + +static void validate_utf8(lua_State *l, struct cfl_variant *v, struct cfl_variant *original, + int depth, int payload) +{ + struct cfl_list *head; + struct cfl_list *original_head; + struct cfl_list *original_end; + struct cfl_kvpair *pair; + struct cfl_kvpair *previous; + struct cfl_variant *old_value; + size_t i; + size_t original_index = 0; + size_t original_count; + int same_key; + + if (values_equal(v, original) || v == NULL) { + return; + } + + if (depth >= FTS_MAX_DEPTH) { + luaL_error(l, "telemetry nesting exceeds limit"); + } + + if (payload && v->type == CFL_VARIANT_UINT && v->data.as_uint64 > INT64_MAX) { + luaL_error(l, "OTel AnyValue integers must fit signed 64 bits"); + } + + if (v->type == CFL_VARIANT_STRING && + memchr(v->data.as_string, '\0', cfl_variant_size_get(v))) { + luaL_error(l, "OTel strings cannot contain NUL in this exporter; use fb.bytes"); + } + + if (v->type == CFL_VARIANT_STRING && + !fts_valid_utf8(v->data.as_string, cfl_variant_size_get(v))) { + luaL_error(l, "OTel strings must be valid UTF-8; use fb.bytes for binary values"); + } + else if (v->type == CFL_VARIANT_KVLIST) { + original_end = original && original->type == CFL_VARIANT_KVLIST ? + &original->data.as_kvlist->list : NULL; + original_head = original_end ? original_end->next : NULL; + + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + previous = NULL; + + /* Set/remove preserve order; match duplicate occurrences positionally. */ + while (original_head != original_end) { + previous = cfl_list_entry(original_head, struct cfl_kvpair, _head); + original_head = original_head->next; + same_key = cfl_sds_len(previous->key) == cfl_sds_len(pair->key) && + memcmp(previous->key, pair->key, cfl_sds_len(pair->key)) == 0; + if (same_key) { + break; + } + previous = NULL; + } + + /* Only new keys need validation; inherited keys were already accepted. */ + if (previous == NULL) { + if (memchr(pair->key, '\0', cfl_sds_len(pair->key))) { + luaL_error(l, "OTel keys cannot contain NUL in this exporter"); + } + if (!fts_valid_utf8(pair->key, cfl_sds_len(pair->key))) { + luaL_error(l, "OTel keys must be valid UTF-8"); + } + } + + validate_utf8(l, pair->val, previous ? previous->val : NULL, depth + 1, payload); + } + } + else if (v->type == CFL_VARIANT_ARRAY) { + original_count = original && original->type == CFL_VARIANT_ARRAY ? + original->data.as_array->entry_count : 0; + + for (i = 0; i < v->data.as_array->entry_count; i++) { + old_value = original_index < original_count ? + original->data.as_array->entries[original_index] : NULL; + + if (old_value && !values_equal(v->data.as_array->entries[i], old_value) && + v->data.as_array->entry_count - i > original_count - original_index) { + /* An inserted value must not shift validation of inherited values. */ + old_value = NULL; + } + else if (old_value) { + original_index++; + } + + validate_utf8(l, v->data.as_array->entries[i], old_value, depth + 1, payload); + } + } +} + +static struct cfl_variant *map_field(struct cfl_variant *map, const char *key) +{ + struct cfl_kvpair *pair; + + if (!map || map->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + pair = fts_find_pair(map, key, strlen(key)); + return pair ? pair->val : NULL; +} + +static struct cfl_variant *log_field(struct cfl_variant *metadata, const char *key) +{ + struct cfl_variant *value; + + value = map_field(map_field(metadata, "otlp"), key); + return value ? value : map_field(metadata, key); +} + +static void push_changed_fields(lua_State *l, struct record_call *call, + struct cfl_variant *metadata, struct cfl_variant *context) +{ + const char *fields[] = {"severity_number", "severity_text", "trace_flags", "trace_id", + "span_id", "event_name", "timestamp", "observed_timestamp", + "attributes", "dropped_attributes_count", NULL}; + const char *context_keys[] = {"resource", "scope", NULL}; + const char *properties[] = {"name", "version", "schema_url", "attributes", + "dropped_attributes_count", NULL}; + struct cfl_variant *current; + struct cfl_variant *original; + char key[64]; + int i; + int j; + + lua_newtable(l); + + /* Log record fields */ + for (i = 0; fields[i]; i++) { + lua_pushboolean(l, !values_equal(log_field(metadata, fields[i]), + log_field(call->record->cobj_metadata->variant, + fields[i]))); + lua_setfield(l, -2, fields[i]); + } + + /* Resource and scope properties, keyed as 'kind.property' */ + for (i = 0; context_keys[i]; i++) { + current = map_field(context, context_keys[i]); + original = map_field(call->attributes->variant, context_keys[i]); + + for (j = 0; properties[j]; j++) { + snprintf(key, sizeof(key), "%s.%s", context_keys[i], properties[j]); + lua_pushboolean(l, !values_equal(map_field(current, properties[j]), + map_field(original, properties[j]))); + lua_setfield(l, -2, key); + } + } +} + +static int empty_context_map(struct cfl_variant *value, int kind) +{ + struct cfl_kvpair *pair; + + if (!value || value->type != CFL_VARIANT_KVLIST) { + return value == NULL; + } + + if (cfl_kvlist_count(value->data.as_kvlist) == 0) { + return FLB_TRUE; + } + + if (kind && cfl_kvlist_count(value->data.as_kvlist) == 1) { + pair = fts_find_pair(value, "attributes", 10); + return pair && empty_context_map(pair->val, FLB_FALSE); + } + + return FLB_FALSE; +} + +static int key_pair_compare(const void *a, const void *b) +{ + const struct cfl_kvpair *left = *(struct cfl_kvpair * const *) a; + const struct cfl_kvpair *right = *(struct cfl_kvpair * const *) b; + size_t left_length = cfl_sds_len(left->key); + size_t right_length = cfl_sds_len(right->key); + + if (left_length != right_length) { + return left_length < right_length ? -1 : 1; + } + + return memcmp(left->key, right->key, left_length); +} + +/* TRUE when a map repeats a key; sorting keeps adversarial maps O(n log n) */ +static int has_duplicate_keys(struct cfl_variant *value) +{ + int count; + int i; + int j; + int duplicate = FLB_FALSE; + struct cfl_list *head; + struct cfl_kvpair **pairs; + + if (!value || value->type != CFL_VARIANT_KVLIST) { + return FLB_FALSE; + } + + count = cfl_kvlist_count(value->data.as_kvlist); + if (count < 2) { + return FLB_FALSE; + } + + pairs = flb_malloc(sizeof(struct cfl_kvpair *) * count); + if (!pairs) { + flb_errno(); + /* without memory to check, compare strictly */ + return FLB_TRUE; + } + + i = 0; + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pairs[i++] = cfl_list_entry(head, struct cfl_kvpair, _head); + } + + if (count <= 8) { + for (i = 0; i < count && !duplicate; i++) { + for (j = i + 1; j < count; j++) { + if (key_pair_compare(&pairs[i], &pairs[j]) == 0) { + duplicate = FLB_TRUE; + break; + } + } + } + } + else { + qsort(pairs, count, sizeof(struct cfl_kvpair *), key_pair_compare); + for (i = 1; i < count; i++) { + if (key_pair_compare(&pairs[i - 1], &pairs[i]) == 0) { + duplicate = FLB_TRUE; + break; + } + } + } + + flb_free(pairs); + return duplicate; +} + +static int pair_key_equal(struct cfl_kvpair *left, struct cfl_kvpair *right) +{ + return cfl_sds_len(left->key) == cfl_sds_len(right->key) && + memcmp(left->key, right->key, cfl_sds_len(left->key)) == 0; +} + +static int empty_attributes_pair(struct cfl_kvpair *pair) +{ + return cfl_sds_len(pair->key) == 10 && memcmp(pair->key, "attributes", 10) == 0 && + empty_context_map(pair->val, FLB_FALSE); +} + +/* + * Positional comparison of a resource or scope map with duplicate keys, where + * matching by name would hide edits to later duplicates. Empty attributes + * placeholders are ignored on both sides. + */ +static int context_kind_equal_positional(struct cfl_variant *left, struct cfl_variant *right) +{ + struct cfl_list *a; + struct cfl_list *b; + struct cfl_list *a_end = &left->data.as_kvlist->list; + struct cfl_list *b_end = &right->data.as_kvlist->list; + struct cfl_kvpair *pa; + struct cfl_kvpair *pb; + + a = a_end->next; + b = b_end->next; + while (1) { + while (a != a_end && empty_attributes_pair(cfl_list_entry(a, struct cfl_kvpair, _head))) { + a = a->next; + } + while (b != b_end && empty_attributes_pair(cfl_list_entry(b, struct cfl_kvpair, _head))) { + b = b->next; + } + if (a == a_end || b == b_end) { + return a == a_end && b == b_end; + } + + pa = cfl_list_entry(a, struct cfl_kvpair, _head); + pb = cfl_list_entry(b, struct cfl_kvpair, _head); + if (!pair_key_equal(pa, pb) || !values_equal(pa->val, pb->val)) { + return FLB_FALSE; + } + a = a->next; + b = b->next; + } +} + +static int context_kind_equal(struct cfl_variant *left, struct cfl_variant *right) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_variant *other; + + if (!left || !right) { + return empty_context_map(left, FLB_TRUE) && empty_context_map(right, FLB_TRUE); + } + + if (left->type != CFL_VARIANT_KVLIST || right->type != CFL_VARIANT_KVLIST) { + return values_equal(left, right); + } + + if (has_duplicate_keys(left) || has_duplicate_keys(right)) { + return context_kind_equal_positional(left, right); + } + + cfl_list_foreach(head, &left->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + other = map_field(right, pair->key); + if (strcmp(pair->key, "attributes") == 0 && + empty_context_map(pair->val, FLB_FALSE) && empty_context_map(other, FLB_FALSE)) { + continue; + } + if (!values_equal(pair->val, other)) { + return FLB_FALSE; + } + } + + cfl_list_foreach(head, &right->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + other = map_field(left, pair->key); + if (!other && + !(strcmp(pair->key, "attributes") == 0 && empty_context_map(pair->val, FLB_FALSE))) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + +static int context_attributes_equal(struct cfl_variant *left, struct cfl_variant *right) +{ + struct cfl_list *head; + struct cfl_list *a; + struct cfl_list *b; + struct cfl_kvpair *pair; + struct cfl_kvpair *pb; + struct cfl_variant *other; + + if (!left || !right || + left->type != CFL_VARIANT_KVLIST || right->type != CFL_VARIANT_KVLIST) { + return values_equal(left, right); + } + + /* duplicate keys are compared by position so later duplicates count */ + if (has_duplicate_keys(left) || has_duplicate_keys(right)) { + a = left->data.as_kvlist->list.next; + b = right->data.as_kvlist->list.next; + while (a != &left->data.as_kvlist->list && b != &right->data.as_kvlist->list) { + pair = cfl_list_entry(a, struct cfl_kvpair, _head); + pb = cfl_list_entry(b, struct cfl_kvpair, _head); + if (!pair_key_equal(pair, pb)) { + return FLB_FALSE; + } + if (strcmp(pair->key, "resource") == 0 || strcmp(pair->key, "scope") == 0) { + if (!context_kind_equal(pair->val, pb->val)) { + return FLB_FALSE; + } + } + else if (!values_equal(pair->val, pb->val)) { + return FLB_FALSE; + } + a = a->next; + b = b->next; + } + return a == &left->data.as_kvlist->list && b == &right->data.as_kvlist->list; + } + + cfl_list_foreach(head, &left->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + other = map_field(right, pair->key); + if (strcmp(pair->key, "resource") == 0 || strcmp(pair->key, "scope") == 0) { + if (!context_kind_equal(pair->val, other)) { + return FLB_FALSE; + } + } + else if (!values_equal(pair->val, other)) { + return FLB_FALSE; + } + } + + cfl_list_foreach(head, &right->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (!map_field(left, pair->key)) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + +static void require_plain_table(lua_State *l, int index, const char *name) +{ + luaL_checktype(l, index, LUA_TTABLE); + if (lua_getmetatable(l, index)) { + lua_pop(l, 1); + luaL_error(l, "%s must not have a metatable", name); + } +} + +static int run_record(lua_State *l) +{ + struct record_call *call; + struct fts_value *original_body; + struct fts_value *original_metadata; + struct fts_value *attributes = NULL; + struct fts_value *metadata = NULL; + struct cfl_object current_metadata; + const char *result; + size_t result_length; + lua_Number sec; + lua_Number nsec; + int event; + int context = 0; + int validator = 0; + int cleanup = 0; + + call = lua_touserdata(l, 1); + + /* Expire record handles from earlier callbacks before creating new ones. */ + call->ctx->generation++; + + /* Build the event table: tag, body, metadata and timestamp */ + lua_createtable(l, 0, 4); + event = lua_gettop(l); + + if (!call->ctx->tag_ready) { + lua_pushlstring(l, call->tag, call->tag_len); + lua_rawseti(l, LUA_REGISTRYINDEX, call->ctx->tag_ref); + call->ctx->tag_ready = FLB_TRUE; + } + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->tag_ref); + fts_setfield(l, event, FTS_KEY_TAG); + + original_body = fts_push_borrowed(l, call->record->cobj_record->variant); + lua_pushvalue(l, -1); + fts_setfield(l, event, FTS_KEY_BODY); + + original_metadata = fts_push_borrowed(l, call->record->cobj_metadata->variant); + lua_pushvalue(l, -1); + fts_setfield(l, event, FTS_KEY_METADATA); + + lua_createtable(l, 0, 2); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_sec); + fts_setfield(l, -2, FTS_KEY_SEC); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_nsec); + fts_setfield(l, -2, FTS_KEY_NSEC); + fts_setfield(l, event, FTS_KEY_TIMESTAMP); + + /* Group context: attributes and metadata are policy-owned clones */ + if (call->attributes) { + lua_createtable(l, 0, 2); + context = lua_gettop(l); + + attributes = push_clone(l, call->attributes->variant); + + /* Pin C-held userdata independently of mutable table fields. */ + lua_pushvalue(l, -1); + fts_setfield(l, context, FTS_KEY_ATTRIBUTES); + + metadata = push_clone(l, call->metadata->variant); + lua_pushvalue(l, -1); + fts_setfield(l, context, FTS_KEY_METADATA); + + lua_pushvalue(l, context); + fts_setfield(l, event, FTS_KEY_CONTEXT); + + /* Semantic aliases are populated by the module without copying data. */ + } + + if (has_otel_context(call->metadata)) { + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->module_ref); + lua_getfield(l, -1, "_attach"); + lua_pushvalue(l, event); + lua_call(l, 1, 2); + cleanup = lua_gettop(l); + validator = cleanup - 1; + } + + original_metadata->dirty = FLB_FALSE; + if (attributes) { + attributes->dirty = FLB_FALSE; + metadata->dirty = FLB_FALSE; + } + + /* Run the policy callback */ + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->policy_ref); + lua_pushvalue(l, event); + lua_call(l, 1, 1); + + if (!lua_isnil(l, -1)) { + result = lua_type(l, -1) == LUA_TSTRING ? lua_tolstring(l, -1, &result_length) : NULL; + if (result && result_length == 4 && memcmp(result, "drop", 4) == 0) { + lua_pushliteral(l, "drop"); + return 1; + } + if (!result || result_length != 4 || memcmp(result, "keep", 4) != 0) { + return luaL_error(l, "callback must return nil, fb.KEEP or fb.DROP"); + } + } + + /* Validation must not execute policy code through table metamethods. */ + require_plain_table(l, event, "log event"); + + /* Roots: metadata and context cannot be replaced */ + fts_getfield(l, event, FTS_KEY_METADATA); + if (lua_touserdata(l, -1) != original_metadata) { + return luaL_error(l, "metadata root cannot be replaced"); + } + lua_pop(l, 1); + + fts_getfield(l, event, FTS_KEY_CONTEXT); + if (context ? !lua_rawequal(l, -1, context) : !lua_isnil(l, -1)) { + return luaL_error(l, "context root cannot be replaced"); + } + lua_pop(l, 1); + + if (context) { + require_plain_table(l, context, "log context"); + } + + validate_map(l, raw_field(l, event, FTS_KEY_BODY), "body"); + validate_map(l, raw_field(l, event, FTS_KEY_METADATA), "metadata"); + + /* Timestamp: validate only when the policy changed it */ + fts_getfield(l, event, FTS_KEY_TIMESTAMP); + require_plain_table(l, -1, "log timestamp"); + fts_getfield(l, -1, FTS_KEY_SEC); + fts_getfield(l, -2, FTS_KEY_NSEC); + if (lua_type(l, -2) != LUA_TNUMBER || lua_type(l, -1) != LUA_TNUMBER) { + return luaL_error(l, "timestamp sec/nsec must be numbers"); + } + + sec = lua_tonumber(l, -2); + nsec = lua_tonumber(l, -1); + if ((sec != (lua_Number) call->record->event.timestamp.tm.tv_sec || + nsec != (lua_Number) call->record->event.timestamp.tm.tv_nsec) && + (!isfinite(sec) || sec < 0 || + sec > (sizeof(time_t) > 4 ? 4294967293.0 : 2147483647.0) || + sec != floor(sec) || !isfinite(nsec) || nsec < 0 || nsec > 999999999 || + nsec != floor(nsec))) { + return luaL_error(l, "timestamp out of range"); + } + lua_pop(l, 3); + + if (validator) { + lua_pushvalue(l, validator); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_sec); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_nsec); + lua_call(l, 2, 0); + } + + /* Context fields may only change through set/remove */ + if (context) { + if (raw_field(l, context, FTS_KEY_ATTRIBUTES) != attributes->owned || + raw_field(l, context, FTS_KEY_METADATA) != metadata->owned) { + return luaL_error(l, "replace context fields through set/remove, not root assignment"); + } + + validate_map(l, raw_field(l, context, FTS_KEY_ATTRIBUTES), "context attributes"); + validate_map(l, raw_field(l, context, FTS_KEY_METADATA), "context metadata"); + + if (!validator) { + memset(¤t_metadata, 0, sizeof(current_metadata)); + current_metadata.variant = metadata->owned; + if (has_otel_context(¤t_metadata)) { + return luaL_error(l, "transport schema cannot be changed"); + } + } + } + + /* OTLP semantic validation */ + if (has_otel_context(call->metadata)) { + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->module_ref); + lua_getfield(l, -1, "_validate"); + lua_pushvalue(l, event); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_sec); + lua_pushnumber(l, call->record->event.timestamp.tm.tv_nsec); + push_changed_fields(l, call, raw_field(l, event, FTS_KEY_METADATA), attributes->owned); + lua_call(l, 4, 0); + + validate_utf8(l, raw_field(l, event, FTS_KEY_BODY), + call->record->cobj_record->variant, 0, FLB_TRUE); + validate_utf8(l, raw_field(l, event, FTS_KEY_METADATA), + call->record->cobj_metadata->variant, 0, FLB_FALSE); + validate_utf8(l, log_field(raw_field(l, event, FTS_KEY_METADATA), "attributes"), + log_field(call->record->cobj_metadata->variant, "attributes"), + 0, FLB_TRUE); + validate_utf8(l, attributes->owned, call->attributes->variant, 0, FLB_TRUE); + validate_utf8(l, metadata->owned, call->metadata->variant, 0, FLB_FALSE); + } + + if (cleanup && lua_istable(l, cleanup) && + (original_metadata->dirty || (attributes && attributes->dirty))) { + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->module_ref); + lua_getfield(l, -1, "_cleanup"); + lua_pushvalue(l, cleanup); + lua_call(l, 1, 0); + lua_pop(l, 1); + } + + /* + * Determine final changes only after all helpers have finished. Results: + * "keep", body, metadata, context attributes, context metadata, sec, nsec. + */ + lua_pushliteral(l, "keep"); + + call->result_body = push_record_result(l, event, FTS_KEY_BODY, original_body); + call->result_metadata = push_record_result(l, event, FTS_KEY_METADATA, original_metadata); + + if (context && attributes->dirty && + !(has_otel_context(call->metadata) ? + context_attributes_equal(attributes->owned, call->attributes->variant) : + values_equal(attributes->owned, call->attributes->variant))) { + push_field_clone(l, context, FTS_KEY_ATTRIBUTES); + } + else { + lua_pushnil(l); + } + + if (context && metadata->dirty && !values_equal(metadata->owned, call->metadata->variant)) { + push_field_clone(l, context, FTS_KEY_METADATA); + } + else { + lua_pushnil(l); + } + + lua_pushnumber(l, sec); + lua_pushnumber(l, nsec); + + return 7; +} + +static struct cfl_object *object_from_variant(struct cfl_variant *v) +{ + struct cfl_object *obj; + + obj = cfl_object_create(); + if (!obj) { + return NULL; + } + + obj->type = CFL_OBJECT_KVLIST; + obj->variant = v; + + return obj; +} + +static struct cfl_object *copy_object(struct cfl_object *source) +{ + struct cfl_variant *v; + struct cfl_object *obj; + + v = fts_copy_value(source->variant, 0); + if (!v) { + return NULL; + } + + obj = object_from_variant(v); + if (!obj) { + cfl_variant_destroy(v); + } + + return obj; +} + +static int values_equal(struct cfl_variant *a, struct cfl_variant *b) +{ + struct cfl_list *left; + struct cfl_list *right; + struct cfl_kvpair *pa; + struct cfl_kvpair *pb; + size_t i; + + if (!a || !b || a == b) { + return a == b; + } + + if (a->type != b->type) { + return FLB_FALSE; + } + + switch (a->type) { + case CFL_VARIANT_NULL: + return FLB_TRUE; + case CFL_VARIANT_BOOL: + return a->data.as_bool == b->data.as_bool; + case CFL_VARIANT_INT: + return a->data.as_int64 == b->data.as_int64; + case CFL_VARIANT_UINT: + return a->data.as_uint64 == b->data.as_uint64; + case CFL_VARIANT_DOUBLE: + return memcmp(&a->data.as_double, &b->data.as_double, sizeof(double)) == 0; + case CFL_VARIANT_STRING: + case CFL_VARIANT_BYTES: + return cfl_variant_size_get(a) == cfl_variant_size_get(b) && + memcmp(a->data.as_string, b->data.as_string, cfl_variant_size_get(a)) == 0; + case CFL_VARIANT_ARRAY: + if (a->data.as_array->entry_count != b->data.as_array->entry_count) { + return FLB_FALSE; + } + for (i = 0; i < a->data.as_array->entry_count; i++) { + if (!values_equal(a->data.as_array->entries[i], b->data.as_array->entries[i])) { + return FLB_FALSE; + } + } + return FLB_TRUE; + case CFL_VARIANT_KVLIST: + left = a->data.as_kvlist->list.next; + right = b->data.as_kvlist->list.next; + while (left != &a->data.as_kvlist->list && right != &b->data.as_kvlist->list) { + pa = cfl_list_entry(left, struct cfl_kvpair, _head); + pb = cfl_list_entry(right, struct cfl_kvpair, _head); + if (cfl_sds_len(pa->key) != cfl_sds_len(pb->key) || + memcmp(pa->key, pb->key, cfl_sds_len(pa->key)) != 0 || + !values_equal(pa->val, pb->val)) { + return FLB_FALSE; + } + left = left->next; + right = right->next; + } + return left == &a->data.as_kvlist->list && right == &b->data.as_kvlist->list; + default: + return FLB_FALSE; + } +} + +static int context_identity_key(struct cfl_kvpair *pair) +{ + return (cfl_sds_len(pair->key) == 11 && memcmp(pair->key, "resource_id", 11) == 0) || + (cfl_sds_len(pair->key) == 8 && memcmp(pair->key, "scope_id", 8) == 0); +} + +/* Compare group metadata ignoring the transport identity keys. */ +static int context_metadata_equal(struct cfl_variant *left, struct cfl_variant *right) +{ + struct cfl_list *a; + struct cfl_list *b; + struct cfl_kvpair *x; + struct cfl_kvpair *y; + + if (left->type != CFL_VARIANT_KVLIST || right->type != CFL_VARIANT_KVLIST) { + return FLB_FALSE; + } + + a = left->data.as_kvlist->list.next; + b = right->data.as_kvlist->list.next; + + for (;;) { + while (a != &left->data.as_kvlist->list && + context_identity_key(cfl_list_entry(a, struct cfl_kvpair, _head))) { + a = a->next; + } + while (b != &right->data.as_kvlist->list && + context_identity_key(cfl_list_entry(b, struct cfl_kvpair, _head))) { + b = b->next; + } + + if (a == &left->data.as_kvlist->list || b == &right->data.as_kvlist->list) { + return a == &left->data.as_kvlist->list && b == &right->data.as_kvlist->list; + } + + x = cfl_list_entry(a, struct cfl_kvpair, _head); + y = cfl_list_entry(b, struct cfl_kvpair, _head); + if (cfl_sds_len(x->key) != cfl_sds_len(y->key) || + memcmp(x->key, y->key, cfl_sds_len(x->key)) != 0 || + !values_equal(x->val, y->val)) { + return FLB_FALSE; + } + + a = a->next; + b = b->next; + } +} + +/* Replace or add a regenerated transport ID; takes ownership of 'value'. */ +static int set_transport_id(struct cfl_variant *metadata, const char *name, + struct cfl_variant *value) +{ + struct cfl_kvpair *pair; + + if (!value) { + return -1; + } + + pair = fts_find_pair(metadata, name, strlen(name)); + if (!pair) { + if (cfl_kvlist_insert(metadata->data.as_kvlist, (char *) name, value) != 0) { + cfl_variant_destroy(value); + return -1; + } + return 0; + } + + cfl_variant_destroy(pair->val); + pair->val = value; + cfl_container_move_variant_to_kvlist(metadata->data.as_kvlist, value); + + return 0; +} + +static struct cfl_object *empty_object(void) +{ + struct cfl_kvlist *map; + struct cfl_variant *v; + struct cfl_object *obj; + + map = cfl_kvlist_create(); + if (!map) { + return NULL; + } + + v = cfl_variant_create_from_kvlist(map); + if (!v) { + cfl_kvlist_destroy(map); + return NULL; + } + + obj = object_from_variant(v); + if (!obj) { + cfl_variant_destroy(v); + } + + return obj; +} + +static int append_marker(struct flb_mp_chunk_cobj *out, int type, + struct cfl_object *attributes, struct cfl_object *metadata, + struct flb_mp_chunk_record **marker) +{ + struct flb_mp_chunk_record *record; + + record = flb_mp_chunk_record_create(out); + if (!record) { + return -1; + } + record->event.timestamp.tm.tv_sec = type; + + if (type == FLB_LOG_EVENT_GROUP_START) { + record->cobj_record = copy_object(attributes); + record->cobj_metadata = copy_object(metadata); + } + else { + /* Later processors expect decoded group ends to carry empty maps. */ + record->cobj_record = empty_object(); + record->cobj_metadata = empty_object(); + } + + if (!record->cobj_record || !record->cobj_metadata) { + flb_mp_chunk_cobj_record_destroy(NULL, record); + return -1; + } + + if (type == FLB_LOG_EVENT_GROUP_START) { + record->cobj_group_attributes = record->cobj_record; + record->cobj_group_metadata = record->cobj_metadata; + } + + cfl_list_add(&record->_head, &out->records); + if (marker) { + *marker = record; + } + + return 0; +} + +/* MessagePack wire size, separate from conservative policy-memory accounting. */ +static size_t wire_value_size(struct cfl_variant *value, int depth) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + size_t size; + size_t child; + size_t count; + size_t index; + int64_t signed_value; + uint64_t unsigned_value; + + if (!value || depth > FTS_MAX_DEPTH) { + return SIZE_MAX; + } + + switch (value->type) { + case CFL_VARIANT_NULL: + case CFL_VARIANT_BOOL: + return 1; + case CFL_VARIANT_DOUBLE: + return 9; + case CFL_VARIANT_INT: + signed_value = value->data.as_int64; + if (signed_value < 0) { + if (signed_value >= -32) { + return 1; + } + else if (signed_value >= INT8_MIN) { + return 2; + } + else if (signed_value >= INT16_MIN) { + return 3; + } + else if (signed_value >= INT32_MIN) { + return 5; + } + return 9; + } + unsigned_value = signed_value; + break; + case CFL_VARIANT_UINT: + unsigned_value = value->data.as_uint64; + break; + case CFL_VARIANT_STRING: + size = cfl_variant_size_get(value); + if (size > SIZE_MAX - 5) { + return SIZE_MAX; + } + if (size < 32) { + return size + 1; + } + else if (size <= UINT8_MAX) { + return size + 2; + } + else if (size <= UINT16_MAX) { + return size + 3; + } + return size + 5; + case CFL_VARIANT_BYTES: + size = cfl_variant_size_get(value); + if (size > SIZE_MAX - 5) { + return SIZE_MAX; + } + if (size <= UINT8_MAX) { + return size + 2; + } + else if (size <= UINT16_MAX) { + return size + 3; + } + return size + 5; + case CFL_VARIANT_ARRAY: + case CFL_VARIANT_KVLIST: + if (value->type == CFL_VARIANT_ARRAY) { + count = value->data.as_array->entry_count; + } + else { + count = cfl_kvlist_count(value->data.as_kvlist); + } + + /* The CFL serializer reserves a map32 header even for small maps. */ + if (value->type == CFL_VARIANT_KVLIST) { + size = 5; + } + else if (count < 16) { + size = 1; + } + else if (count <= UINT16_MAX) { + size = 3; + } + else { + size = 5; + } + + if (value->type == CFL_VARIANT_ARRAY) { + for (index = 0; index < count; index++) { + child = wire_value_size(value->data.as_array->entries[index], depth + 1); + if (child > SIZE_MAX - size) { + return SIZE_MAX; + } + size += child; + } + } + else { + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + + /* key: string header plus bytes */ + child = cfl_sds_len(pair->key); + if (size > SIZE_MAX - 5 || child > SIZE_MAX - size - 5) { + return SIZE_MAX; + } + if (child < 32) { + size += child + 1; + } + else if (child <= UINT8_MAX) { + size += child + 2; + } + else if (child <= UINT16_MAX) { + size += child + 3; + } + else { + size += child + 5; + } + + /* value */ + child = wire_value_size(pair->val, depth + 1); + if (child > SIZE_MAX - size) { + return SIZE_MAX; + } + size += child; + } + } + return size; + default: + return SIZE_MAX; + } + + /* Non-negative integers */ + if (unsigned_value <= 127) { + return 1; + } + else if (unsigned_value <= UINT8_MAX) { + return 2; + } + else if (unsigned_value <= UINT16_MAX) { + return 3; + } + else if (unsigned_value <= UINT32_MAX) { + return 5; + } + return 9; +} + +struct borrowed_record { + struct flb_mp_chunk_record *source; + struct flb_mp_chunk_record *result; + int body; + int metadata; + int pooled; + struct cfl_list _head; +}; + +#define BORROW_BLOCK_ENTRIES 64 +#define BORROW_BLOCK_LIMIT 8 + +struct borrowed_record_block { + struct borrowed_record_block *next; + size_t used; + struct borrowed_record records[BORROW_BLOCK_ENTRIES]; +}; + +struct borrowed_record_pool { + struct borrowed_record_block *blocks; + size_t count; +}; + +/* This pool owns bookkeeping only, never policy values or borrowed objects. */ +static struct borrowed_record *borrow_record_create(struct borrowed_record_pool *pool) +{ + struct borrowed_record_block *block; + struct borrowed_record *record; + + block = pool->blocks; + if ((!block || block->used == BORROW_BLOCK_ENTRIES) && pool->count < BORROW_BLOCK_LIMIT) { + /* On failure fall back to an individual heap allocation below */ + block = flb_calloc(1, sizeof(struct borrowed_record_block)); + if (block) { + block->next = pool->blocks; + pool->blocks = block; + pool->count++; + } + } + + if (block && block->used < BORROW_BLOCK_ENTRIES) { + record = &block->records[block->used++]; + record->pooled = FLB_TRUE; + return record; + } + + record = flb_calloc(1, sizeof(struct borrowed_record)); + if (!record) { + flb_errno(); + } + + return record; +} + +/* Ownership moves only after the entire staged chunk has been constructed. */ +static void release_borrowed_records(struct cfl_list *records, + struct borrowed_record_pool *pool, int commit) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct borrowed_record *record; + struct borrowed_record_block *block; + + cfl_list_foreach_safe(head, tmp, records) { + record = cfl_list_entry(head, struct borrowed_record, _head); + + if (record->body) { + if (commit) { + record->source->cobj_record = NULL; + } + else { + record->result->cobj_record = NULL; + } + } + + if (record->metadata) { + if (commit) { + record->source->cobj_metadata = NULL; + } + else { + record->result->cobj_metadata = NULL; + } + } + + cfl_list_del(head); + if (!record->pooled) { + flb_free(record); + } + } + + while (pool->blocks) { + block = pool->blocks; + pool->blocks = block->next; + flb_free(block); + } + pool->count = 0; +} + +/* + * A following filter forces serialization; a following native processor may + * mutate CFL containers and therefore requires ordinary heap-backed values. + */ +static int serializes_after_callback(struct flb_processor_instance *ins) +{ + struct flb_processor *parent = ins->pu->parent; + struct mk_list *next = ins->pu->_head.next; + struct flb_processor_unit *unit; + + if (next == &parent->logs) { + return FLB_TRUE; + } + + unit = mk_list_entry(next, struct flb_processor_unit, _head); + return unit->unit_type == FLB_PROCESSOR_UNIT_FILTER; +} + +/* + * Open a group with the source context of a group that continues in the next + * append, charging it to the output limit. OTLP transport IDs are fresh, so + * the group never merges with an earlier group of this chunk. + */ +static int reopen_source_group(struct fts_runtime *ctx, struct flb_mp_chunk_cobj *out, + struct cfl_object *source_attrs, + struct cfl_object *source_meta, + int resource_id, int scope_id, size_t *output_bytes) +{ + size_t cost; + struct cfl_object attrs = {0}; + struct cfl_object meta = {0}; + struct cfl_variant *id_value; + struct flb_mp_chunk_record *group = NULL; + + cost = 32; + if (*output_bytes > ctx->output_limit || ctx->output_limit - *output_bytes < cost) { + return -1; + } + *output_bytes += cost; + + attrs.type = CFL_OBJECT_KVLIST; + attrs.variant = source_attrs->variant; + meta.type = CFL_OBJECT_KVLIST; + meta.variant = source_meta ? source_meta->variant : NULL; + + cost = wire_value_size(attrs.variant, 0); + if (cost > ctx->output_limit || *output_bytes > ctx->output_limit - cost) { + return -1; + } + *output_bytes += cost; + + cost = wire_value_size(meta.variant, 0); + if (cost > ctx->output_limit || *output_bytes > ctx->output_limit - cost) { + return -1; + } + *output_bytes += cost; + + if (append_marker(out, FLB_LOG_EVENT_GROUP_START, &attrs, &meta, &group) != 0) { + return -1; + } + + if (has_otel_context(group->cobj_metadata)) { + id_value = cfl_variant_create_from_int64(resource_id); + if (set_transport_id(group->cobj_metadata->variant, "resource_id", id_value) != 0) { + return -1; + } + id_value = cfl_variant_create_from_int64(scope_id); + if (set_transport_id(group->cobj_metadata->variant, "scope_id", id_value) != 0) { + return -1; + } + } + + return 0; +} + +int fts_process_logs(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + struct cfl_list borrowed; + struct borrowed_record_pool borrow_pool = {0}; + struct borrowed_record *borrow; + struct fts_runtime *ctx = ins->context; + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_cobj *out = NULL; + struct flb_mp_chunk_record *record; + struct flb_mp_chunk_record *next; + struct flb_mp_chunk_record *group = NULL; + struct flb_mp_chunk_record *last_group = NULL; + struct cfl_variant *last_source_id = NULL; + struct cfl_kvpair *source_id; + struct cfl_kvpair *last_resource; + struct cfl_kvpair *current_resource; + struct cfl_kvpair *last_identity; + int resource_id = 0; + struct cfl_object *source_attrs = NULL; + struct cfl_object *source_meta = NULL; + struct cfl_object attrs; + struct cfl_object meta; + struct record_call call; + struct flb_condition *condition; + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_variant *body; + struct cfl_variant *metadata; + struct fts_value *context_attrs; + struct fts_value *context_meta; + lua_State *l = ctx->vm->state; + int ret; + int type; + int group_number = 0; + int emitted_group_number = -1; + int selected; + int base; + int emitted_id = 0; + int errors = 0; + int source_open = FLB_FALSE; + struct flb_mp_chunk_record *continuation_end = NULL; + size_t undo_mark; + struct cfl_variant *id_value; + size_t output_bytes = 0; + size_t cost; + + ctx->tag_ready = FLB_FALSE; + cfl_list_init(&borrowed); + + chunk->record_pool_enabled = FLB_TRUE; + if (serializes_after_callback(ins)) { + flb_mp_chunk_cobj_value_pool_enable(chunk); + } + + /* Decode every record of the chunk, ignoring the condition for now */ + condition = chunk->condition; + chunk->condition = NULL; + while ((ret = flb_mp_chunk_cobj_record_next(chunk, &record)) == FLB_MP_CHUNK_RECORD_OK) { + } + chunk->condition = condition; + chunk->record_pos = NULL; + if (ret != FLB_MP_CHUNK_RECORD_EOF) { + goto failure; + } + + /* Staged output chunk; it replaces the original records on success */ + out = flb_calloc(1, sizeof(struct flb_mp_chunk_cobj)); + if (!out) { + flb_errno(); + goto failure; + } + cfl_list_init(&out->records); + out->record_pool_enabled = FLB_TRUE; + + /* + * Records before an end marker that precedes any start belong to a group + * opened by a previous append: they keep no context and the end marker is + * forwarded to close that group. + */ + cfl_list_foreach(head, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + if (flb_log_event_decoder_get_record_type(&record->event, &type) != 0) { + goto failure; + } + if (type == FLB_LOG_EVENT_GROUP_END) { + continuation_end = record; + break; + } + if (type == FLB_LOG_EVENT_GROUP_START) { + break; + } + } + + cfl_list_foreach(head, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + + /* Group markers update the source context and are not copied as-is */ + if (flb_log_event_decoder_get_record_type(&record->event, &type) != 0) { + goto failure; + } + + if (type == FLB_LOG_EVENT_GROUP_START) { + /* A nested start establishes a new flat transport context. */ + source_attrs = record->cobj_record; + source_meta = record->cobj_metadata; + source_open = FLB_TRUE; + group_number++; + continue; + } + + if (type == FLB_LOG_EVENT_GROUP_END) { + if (record == continuation_end) { + if (output_bytes > ctx->output_limit || ctx->output_limit - output_bytes < 32) { + goto failure; + } + output_bytes += 32; + if (append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + goto failure; + } + continuation_end = NULL; + continue; + } + + /* Stray ends carry no record and do not invalidate healthy siblings. */ + source_attrs = NULL; + source_meta = NULL; + source_open = FLB_FALSE; + continue; + } + + /* Defaults for records that are not selected or keep their values */ + selected = !condition || flb_condition_evaluate(condition, record); + memset(&attrs, 0, sizeof(attrs)); + memset(&meta, 0, sizeof(meta)); + attrs.variant = source_attrs ? source_attrs->variant : NULL; + meta.variant = source_meta ? source_meta->variant : NULL; + base = lua_gettop(l); + ret = 0; + body = NULL; + metadata = NULL; + + /* Run the policy for selected records */ + if (selected) { + memset(&call, 0, sizeof(call)); + call.ctx = ctx; + call.chunk = chunk; + call.record = record; + call.attributes = source_attrs; + call.metadata = source_meta; + call.tag = tag; + call.tag_len = tag_len; + + /* + * Plain records edit the chunk's heap tree in place; the undo log + * keeps the whole chunk reversible until its output is final. + */ + undo_mark = ctx->undo_count; + ctx->inplace_ok = !source_attrs && ctx->undo_held <= ctx->memory_limit; + ret = fts_runtime_call(ctx, &call, 7); + ctx->inplace_ok = FLB_FALSE; + ctx->native_bytes -= ctx->undo_charged; + ctx->undo_charged = 0; + + if (ret != 0) { + fts_undo_rollback(ctx, undo_mark); + errors++; + if (errors <= 5) { + flb_plg_error(ins, "log policy failed: %.256s; edits rolled back", + lua_type(l, -1) == LUA_TSTRING ? + lua_tostring(l, -1) : "non-string error"); + } + if (strcmp(ctx->error_mode, "drop") == 0) { + lua_settop(l, base); + continue; + } + } + else if (lua_type(l, base + 1) == LUA_TSTRING && + strcmp(lua_tostring(l, base + 1), "drop") == 0) { + lua_settop(l, base); + continue; + } + else { + body = call.result_body; + metadata = call.result_metadata; + context_attrs = lua_touserdata(l, base + 4); + context_meta = lua_touserdata(l, base + 5); + if (context_attrs) { + attrs.variant = context_attrs->owned; + } + if (context_meta) { + meta.variant = context_meta->owned; + } + } + + /* a group context would split the group continued from a previous append */ + if (continuation_end && attrs.variant) { + fts_undo_rollback(ctx, undo_mark); + errors++; + if (errors <= 5) { + flb_plg_error(ins, "log policy failed: records continuing a group " + "from a previous chunk cannot set a context; " + "edits rolled back"); + } + if (strcmp(ctx->error_mode, "drop") == 0) { + lua_settop(l, base); + continue; + } + ret = -1; + body = NULL; + metadata = NULL; + attrs.variant = NULL; + meta.variant = NULL; + } + } + + /* Close the open group when the record's context no longer matches it */ + if (group && (!attrs.variant || emitted_group_number != group_number || + !values_equal(group->cobj_record->variant, attrs.variant) || + !(has_otel_context(group->cobj_metadata) ? + context_metadata_equal(group->cobj_metadata->variant, meta.variant) : + values_equal(group->cobj_metadata->variant, meta.variant)))) { + if (append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + lua_settop(l, base); + goto failure; + } + group = NULL; + } + + /* Open a new group, charging its marker and context to the output limit */ + if (attrs.variant && !group) { + if (output_bytes > ctx->output_limit || ctx->output_limit - output_bytes < 32) { + lua_settop(l, base); + goto failure; + } + output_bytes += 32; + + cost = wire_value_size(attrs.variant, 0); + if (cost > ctx->output_limit || output_bytes > ctx->output_limit - cost) { + lua_settop(l, base); + goto failure; + } + output_bytes += cost; + + cost = wire_value_size(meta.variant, 0); + if (cost > ctx->output_limit || output_bytes > ctx->output_limit - cost) { + lua_settop(l, base); + goto failure; + } + output_bytes += cost; + + attrs.type = CFL_OBJECT_KVLIST; + meta.type = CFL_OBJECT_KVLIST; + if (append_marker(out, FLB_LOG_EVENT_GROUP_START, &attrs, &meta, &group) != 0) { + lua_settop(l, base); + goto failure; + } + + /* OTLP groups get regenerated resource_id/scope_id transport IDs */ + if (has_otel_context(group->cobj_metadata)) { + /* Groups without a source ID never share a resource with others. */ + source_id = fts_find_pair(source_meta->variant, "resource_id", 11); + last_resource = last_group ? + fts_find_pair(last_group->cobj_record->variant, "resource", 8) : + NULL; + current_resource = fts_find_pair(group->cobj_record->variant, "resource", 8); + + if (source_id && last_group && last_source_id && + values_equal(last_source_id, source_id->val) && + context_kind_equal(last_resource ? last_resource->val : NULL, + current_resource ? current_resource->val : NULL)) { + last_identity = fts_find_pair(last_group->cobj_metadata->variant, + "resource_id", 11); + id_value = fts_copy_value(last_identity->val, 0); + } + else { + id_value = cfl_variant_create_from_int64(resource_id++); + } + + if (set_transport_id(group->cobj_metadata->variant, "resource_id", + id_value) != 0) { + lua_settop(l, base); + goto failure; + } + + id_value = cfl_variant_create_from_int64(emitted_id++); + if (set_transport_id(group->cobj_metadata->variant, "scope_id", id_value) != 0) { + lua_settop(l, base); + goto failure; + } + + last_group = group; + last_source_id = source_id ? source_id->val : NULL; + } + else { + last_group = NULL; + last_source_id = NULL; + } + emitted_group_number = group_number; + } + + if (!attrs.variant) { + last_group = NULL; + last_source_id = NULL; + } + + /* Charge the record framing, body and metadata to the output limit */ + if (output_bytes > ctx->output_limit || ctx->output_limit - output_bytes < 16) { + lua_settop(l, base); + goto failure; + } + output_bytes += 16; + + cost = wire_value_size(body ? body : record->cobj_record->variant, 0); + if (cost > ctx->output_limit || output_bytes > ctx->output_limit - cost) { + lua_settop(l, base); + goto failure; + } + output_bytes += cost; + + cost = wire_value_size(metadata ? metadata : record->cobj_metadata->variant, 0); + if (cost > ctx->output_limit || output_bytes > ctx->output_limit - cost) { + lua_settop(l, base); + goto failure; + } + output_bytes += cost; + + /* + * Build the output record: changed roots are copied into the chunk, + * unchanged roots are borrowed from the source record. + */ + next = flb_mp_chunk_record_create(out); + if (!next) { + lua_settop(l, base); + goto failure; + } + next->event = record->event; + next->cobj_record = body ? result_object(chunk, body) : record->cobj_record; + next->cobj_metadata = metadata ? result_object(chunk, metadata) : record->cobj_metadata; + + if (selected && ret == 0 && + (record->event.timestamp.tm.tv_sec != (time_t) lua_tonumber(l, base + 6) || + record->event.timestamp.tm.tv_nsec != (long) lua_tonumber(l, base + 7))) { + next->event.timestamp.tm.tv_sec = (time_t) lua_tonumber(l, base + 6); + next->event.timestamp.tm.tv_nsec = (long) lua_tonumber(l, base + 7); + next->event.raw_timestamp = NULL; + } + + if (!next->cobj_record || !next->cobj_metadata) { + if (!body) { + next->cobj_record = NULL; + } + if (!metadata) { + next->cobj_metadata = NULL; + } + flb_mp_chunk_cobj_record_destroy(NULL, next); + lua_settop(l, base); + goto failure; + } + + /* Track borrowed roots so ownership can be settled at commit/rollback */ + if (!body || !metadata) { + borrow = borrow_record_create(&borrow_pool); + if (!borrow) { + if (!body) { + next->cobj_record = NULL; + } + if (!metadata) { + next->cobj_metadata = NULL; + } + flb_mp_chunk_cobj_record_destroy(NULL, next); + lua_settop(l, base); + goto failure; + } + borrow->source = record; + borrow->result = next; + borrow->body = !body; + borrow->metadata = !metadata; + cfl_list_add(&borrow->_head, &borrowed); + } + + next->cobj_group_attributes = group ? group->cobj_record : NULL; + next->cobj_group_metadata = group ? group->cobj_metadata : NULL; + cfl_list_add(&next->_head, &out->records); + lua_settop(l, base); + } + + /* + * A source group still open at the end continues in the next append: keep + * the output group open when it carries the source context, otherwise close + * it and reopen the source context for the records that follow. + */ + if (source_open && source_attrs && group && + (emitted_group_number != group_number || + !values_equal(group->cobj_record->variant, source_attrs->variant) || + !(has_otel_context(group->cobj_metadata) ? + context_metadata_equal(group->cobj_metadata->variant, source_meta->variant) : + values_equal(group->cobj_metadata->variant, source_meta->variant)))) { + if (append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + goto failure; + } + group = NULL; + } + + if (source_open && source_attrs) { + if (!group) { + if (reopen_source_group(ctx, out, source_attrs, source_meta, resource_id, + emitted_id, &output_bytes) != 0) { + goto failure; + } + } + } + else if (group && append_marker(out, FLB_LOG_EVENT_GROUP_END, NULL, NULL, NULL) != 0) { + /* Close an unterminated transport group after its last retained record. */ + goto failure; + } + + /* Commit: drop the source records and move the staged ones into the chunk */ + fts_undo_commit(ctx); + release_borrowed_records(&borrowed, &borrow_pool, FLB_TRUE); + + cfl_list_foreach_safe(head, tmp, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + flb_mp_chunk_cobj_record_destroy(NULL, record); + } + + cfl_list_foreach_safe(head, tmp, &out->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + cfl_list_del(head); + cfl_list_add(head, &chunk->records); + } + + chunk->active_group_attributes = NULL; + chunk->active_group_metadata = NULL; + flb_mp_chunk_cobj_destroy(out); + + if (errors) { + fts_report_policy_error(ins); + if (errors > 5) { + flb_plg_warn(ins, "%d additional policy errors in this chunk", errors - 5); + } + } + + ctx->instructions_left = ctx->instruction_limit; + fts_collect(ctx, FLB_FALSE); + + return FLB_PROCESSOR_SUCCESS; + +failure: + fts_undo_rollback(ctx, 0); + release_borrowed_records(&borrowed, &borrow_pool, FLB_FALSE); + flb_mp_chunk_cobj_destroy(out); + ctx->instructions_left = ctx->instruction_limit; + fts_collect(ctx, FLB_FALSE); + + flb_plg_error(ins, "cannot construct policy output; edits discarded"); + if (strcmp(ctx->error_mode, "drop") == 0) { + cfl_list_foreach_safe(head, tmp, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + flb_mp_chunk_cobj_record_destroy(NULL, record); + } + chunk->active_group_attributes = NULL; + chunk->active_group_metadata = NULL; + fts_report_policy_error(ins); + return FLB_PROCESSOR_SUCCESS; + } + + return FLB_PROCESSOR_FAILURE; +} + +static int register_log_helpers(lua_State *l) +{ + if (fts_load_helpers(l, fts_log_helpers, sizeof(fts_log_helpers) - 1, + "fluentbit.log_helpers") != 0) { + return -1; + } + + return 0; +} + +const struct fts_signal_adapter fts_logs_adapter = { + .event_type = FLB_PROCESSOR_LOGS, + .register_helpers = register_log_helpers, + .run_callback = run_record +}; diff --git a/plugins/processor_fts/fts_metrics.c b/plugins/processor_fts/fts_metrics.c new file mode 100644 index 00000000000..eeff0cbd919 --- /dev/null +++ b/plugins/processor_fts/fts_metrics.c @@ -0,0 +1,615 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fts_telemetry.h" +#include + +const struct fts_signal_adapter fts_metrics_adapter = { + .event_type = FLB_PROCESSOR_METRICS, + .register_helpers = fts_register_telemetry_helpers, + .run_callback = fts_run_telemetry +}; + +static struct cfl_kvlist *map_at(struct cfl_kvlist *map, char *key) +{ + struct cfl_variant *value; + struct cfl_kvlist *child; + + value = cfl_kvlist_fetch(map, key); + if (value) { + return value->type == CFL_VARIANT_KVLIST ? value->data.as_kvlist : NULL; + } + + child = cfl_kvlist_create(); + if (!child || cfl_kvlist_insert_kvlist(map, key, child) != 0) { + cfl_kvlist_destroy(child); + return NULL; + } + + return child; +} + +static int copy_map(struct cfl_kvlist *dst, struct cfl_kvlist *src) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + struct cfl_variant *copy; + + cfl_list_foreach(head, &src->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + copy = fts_copy_value(pair->val, 0); + if (!copy || cfl_kvlist_insert_s(dst, pair->key, cfl_sds_len(pair->key), copy) != 0) { + cfl_variant_destroy(copy); + return -1; + } + } + + return 0; +} + +static int family_metadata(struct cmt *out, struct cmt *source, struct cmt_map *family, + const char *type, size_t resource_index) +{ + struct cfl_kvlist *metadata; + + if (!family->otlp_metadata) { + family->otlp_metadata = cfl_kvlist_create(); + } + + if (!family->otlp_metadata) { + return -1; + } + + metadata = map_at(family->otlp_metadata, "routing"); + if (!metadata) { + return -1; + } + + cfl_kvlist_remove(metadata, "scope_flat_index"); + cfl_kvlist_remove(metadata, "resource_index"); + cfl_kvlist_remove(metadata, "scope_index"); + + if (cfl_kvlist_insert_uint64(metadata, "resource_index", resource_index) != 0 || + cfl_kvlist_insert_uint64(metadata, "scope_index", 0) != 0) { + return -1; + } + + return 0; +} + +/* Move complete families: concatenation would aggregate samples and lose identity. */ +#define MOVE_FAMILIES(member, family_type, family, type_name) \ + cfl_list_foreach_safe(head, tmp, &source->member) { \ + family = cfl_list_entry(head, family_type, _head); \ + if (family_metadata(out, source, family->map, type_name, index) != 0) { \ + goto failure; \ + } \ + cfl_list_del(head); \ + cfl_list_add(head, &out->member); \ + family->cmt = out; \ + } + +static struct cmt *merge_contexts(struct cfl_list *contexts, struct cmt *input) +{ + struct cmt *out; + struct cmt *source; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; + struct cmt_summary *summary; + struct cmt_untyped *untyped; + struct cfl_list *context_head; + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_array *resources; + struct cfl_kvlist *resource; + struct cfl_variant *value; + struct cfl_variant *copy; + const char *keys[] = {"resource", "resource_metrics", "scope", "scope_metrics"}; + size_t i; + size_t index = 0; + + out = cmt_create(); + if (!out) { + return NULL; + } + + if (copy_map(out->internal_metadata, input->internal_metadata) != 0 || + copy_map(out->external_metadata, input->external_metadata) != 0) { + goto failure; + } + + cfl_kvlist_remove(out->external_metadata, "resource_metrics_list"); + cfl_kvlist_remove(out->external_metadata, "otlp"); + + resources = cfl_array_create(cfl_list_size(contexts)); + if (!resources || + cfl_kvlist_insert_array(out->external_metadata, "resource_metrics_list", + resources) != 0) { + cfl_array_destroy(resources); + goto failure; + } + + cfl_list_foreach(context_head, contexts) { + source = cfl_list_entry(context_head, struct cmt, _head); + + resource = cfl_kvlist_create(); + if (!resource || cfl_array_append_kvlist(resources, resource) != 0) { + cfl_kvlist_destroy(resource); + goto failure; + } + + for (i = 0; i < sizeof(keys) / sizeof(keys[0]); i++) { + value = cfl_kvlist_fetch(source->external_metadata, (char *) keys[i]); + if (value) { + copy = fts_copy_value(value, 0); + if (!copy || cfl_kvlist_insert(resource, (char *) keys[i], copy) != 0) { + cfl_variant_destroy(copy); + goto failure; + } + } + } + + MOVE_FAMILIES(counters, struct cmt_counter, counter, "counter"); + MOVE_FAMILIES(gauges, struct cmt_gauge, gauge, "gauge"); + MOVE_FAMILIES(histograms, struct cmt_histogram, histogram, "histogram"); + MOVE_FAMILIES(exp_histograms, struct cmt_exp_histogram, exp_histogram, "exp_histogram"); + MOVE_FAMILIES(summaries, struct cmt_summary, summary, "summary"); + MOVE_FAMILIES(untypeds, struct cmt_untyped, untyped, "untyped"); + index++; + } + + return out; + +failure: + cmt_destroy(out); + return NULL; +} + +/* OTLP scalars are int64 or double: uint64 values must fit int64 exactly. */ +static int scalar_value_supported(struct cmt_metric *sample) +{ + if (cmt_metric_get_value_type(sample) != CMT_METRIC_VALUE_UINT64) { + return FLB_TRUE; + } + + return cmt_metric_get_uint64_value(sample) <= INT64_MAX; +} + +static int scalar_values_supported(struct cmt_map *map) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + if (map->metric_static_set && !scalar_value_supported(&map->metric)) { + return FLB_FALSE; + } + + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + if (!scalar_value_supported(sample)) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + +struct unsupported_family { + struct cfl_list *head; + struct cfl_list *next; + struct cmt_map *map; + void *family; + int type; + struct cfl_kvlist *metadata; +}; + +/* Families are detached only while the input is encoded, then restored. */ +#define DETACH_UNSUPPORTED(member, family_type, item, map_type) \ + cfl_list_foreach_safe(head, next, &input->member) { \ + item = cfl_list_entry(head, family_type, _head); \ + if (!scalar_values_supported(item->map)) { \ + rejected[rejected_count].head = head; \ + rejected[rejected_count].next = head->next; \ + rejected[rejected_count].map = item->map; \ + rejected[rejected_count].family = item; \ + rejected[rejected_count].type = map_type; \ + rejected_count++; \ + cfl_list_del(head); \ + } \ + } + +static uint64_t routing_value(struct cfl_kvlist *routing, char *key) +{ + struct cfl_variant *value; + + value = routing ? cfl_kvlist_fetch(routing, key) : NULL; + if (value && value->type == CFL_VARIANT_UINT) { + return value->data.as_uint64; + } + + if (value && value->type == CFL_VARIANT_INT && value->data.as_int64 >= 0) { + return (uint64_t) value->data.as_int64; + } + + return 0; +} + +/* Preserve the original contexts of families that cannot enter the OTLP API. */ +static int prepare_rejected_families(struct cmt *out, struct cmt *input, + struct unsupported_family *families, size_t count) +{ + struct cfl_variant *output_resources; + struct cfl_variant *input_resources; + struct cfl_variant *copy; + struct cfl_variant *scopes; + struct cfl_variant *flat; + struct cfl_kvlist *resource; + struct cfl_kvlist *routing; + const char *keys[] = {"resource", "resource_metrics", "scope", "scope_metrics"}; + size_t offset; + size_t index; + size_t key_index; + size_t resource_index; + size_t scope_index; + uint64_t flat_index; + size_t scope_count; + + output_resources = cfl_kvlist_fetch(out->external_metadata, "resource_metrics_list"); + if (!output_resources || output_resources->type != CFL_VARIANT_ARRAY) { + return -1; + } + + cfl_array_resizable(output_resources->data.as_array, CFL_TRUE); + offset = output_resources->data.as_array->entry_count; + + input_resources = cfl_kvlist_fetch(input->external_metadata, "resource_metrics_list"); + if (input_resources && input_resources->type == CFL_VARIANT_ARRAY && + input_resources->data.as_array->entry_count > 0) { + for (index = 0; index < input_resources->data.as_array->entry_count; index++) { + copy = fts_copy_value(input_resources->data.as_array->entries[index], 0); + if (!copy || cfl_array_append(output_resources->data.as_array, copy) != 0) { + cfl_variant_destroy(copy); + return -1; + } + } + } + else { + resource = cfl_kvlist_create(); + if (!resource || cfl_array_append_kvlist(output_resources->data.as_array, resource) != 0) { + cfl_kvlist_destroy(resource); + return -1; + } + + for (key_index = 0; key_index < sizeof(keys) / sizeof(keys[0]); key_index++) { + copy = cfl_kvlist_fetch(input->external_metadata, (char *) keys[key_index]); + if (copy) { + copy = fts_copy_value(copy, 0); + if (!copy || cfl_kvlist_insert(resource, (char *) keys[key_index], copy) != 0) { + cfl_variant_destroy(copy); + return -1; + } + } + } + } + + for (index = 0; index < count; index++) { + if (families[index].map->otlp_metadata) { + families[index].metadata = + cmt_otlp_metadata_clone(families[index].map->otlp_metadata); + } + else { + families[index].metadata = cfl_kvlist_create(); + } + + if (!families[index].metadata) { + return -1; + } + + routing = map_at(families[index].metadata, "routing"); + if (!routing) { + return -1; + } + + resource_index = routing_value(routing, "resource_index"); + scope_index = routing_value(routing, "scope_index"); + + flat = cfl_kvlist_fetch(routing, "scope_flat_index"); + if (flat && input_resources && input_resources->type == CFL_VARIANT_ARRAY) { + flat_index = routing_value(routing, "scope_flat_index"); + for (resource_index = 0; + resource_index < input_resources->data.as_array->entry_count; + resource_index++) { + copy = input_resources->data.as_array->entries[resource_index]; + if (copy->type == CFL_VARIANT_KVLIST) { + scopes = cfl_kvlist_fetch(copy->data.as_kvlist, "scope_metrics_list"); + } + else { + scopes = NULL; + } + + if (scopes && scopes->type == CFL_VARIANT_ARRAY && + scopes->data.as_array->entry_count) { + scope_count = scopes->data.as_array->entry_count; + } + else { + scope_count = 1; + } + + if (flat_index < scope_count) { + scope_index = flat_index; + break; + } + flat_index -= scope_count; + } + } + + if (resource_index >= output_resources->data.as_array->entry_count - offset) { + resource_index = 0; + scope_index = 0; + } + + cfl_kvlist_remove(routing, "scope_flat_index"); + cfl_kvlist_remove(routing, "resource_index"); + cfl_kvlist_remove(routing, "scope_index"); + + if (cfl_kvlist_insert_uint64(routing, "resource_index", offset + resource_index) != 0 || + cfl_kvlist_insert_uint64(routing, "scope_index", scope_index) != 0) { + return -1; + } + } + + return 0; +} + +static int family_name_used(struct cmt *out, int type, const char *fqname) +{ + struct cfl_list *list; + struct cfl_list *head; + struct cmt_map *map; + + if (type == CMT_COUNTER) { + list = &out->counters; + } + else if (type == CMT_GAUGE) { + list = &out->gauges; + } + else { + list = &out->untypeds; + } + + cfl_list_foreach(head, list) { + if (type == CMT_COUNTER) { + map = cfl_list_entry(head, struct cmt_counter, _head)->map; + } + else if (type == CMT_GAUGE) { + map = cfl_list_entry(head, struct cmt_gauge, _head)->map; + } + else { + map = cfl_list_entry(head, struct cmt_untyped, _head)->map; + } + if (strcmp(map->opts->fqname, fqname) == 0) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + +/* The input context stays owned by its producer, so kept families are copied. */ +static int copy_rejected_families(struct cmt *out, struct unsupported_family *families, + size_t count) +{ + int ret; + size_t index; + struct cfl_list *list; + struct cfl_list *head; + struct cmt_map *map; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + + for (index = 0; index < count; index++) { + /* + * cmt_cat merges into an existing family with the same name: a policy + * output family must never absorb the kept samples and their routing. + */ + if (family_name_used(out, families[index].type, + families[index].map->opts->fqname)) { + return -1; + } + + if (families[index].type == CMT_COUNTER) { + ret = cmt_cat_counter(out, families[index].family, NULL); + list = &out->counters; + } + else if (families[index].type == CMT_GAUGE) { + ret = cmt_cat_gauge(out, families[index].family, NULL); + list = &out->gauges; + } + else { + ret = cmt_cat_untyped(out, families[index].family, NULL); + list = &out->untypeds; + } + + if (ret != 0) { + return -1; + } + + /* Attach the routing metadata prepared for this family to its copy. */ + map = NULL; + cfl_list_foreach(head, list) { + if (families[index].type == CMT_COUNTER) { + counter = cfl_list_entry(head, struct cmt_counter, _head); + map = counter->map; + } + else if (families[index].type == CMT_GAUGE) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + map = gauge->map; + } + else { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + map = untyped->map; + } + + if (strcmp(map->opts->fqname, families[index].map->opts->fqname) == 0) { + break; + } + map = NULL; + } + + if (!map) { + return -1; + } + + cfl_kvlist_destroy(map->otlp_metadata); + map->otlp_metadata = families[index].metadata; + families[index].metadata = NULL; + } + + return 0; +} + +int fts_process_metrics(struct flb_processor_instance *ins, struct cmt *input, + struct cmt **output, const char *tag, int tag_len) +{ + struct fts_runtime *ctx = ins->context; + cfl_sds_t encoded; + char *buffer = NULL; + size_t size; + size_t offset = 0; + struct cfl_list contexts; + struct cfl_list *head; + struct cfl_list *next; + struct unsupported_family *rejected = NULL; + size_t rejected_count = 0; + size_t family_count; + size_t index; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + int ret; + int handled_errors; + const char *phase = "encode native metrics"; + + *output = NULL; + cfl_list_init(&contexts); + + family_count = cfl_list_size(&input->counters) + cfl_list_size(&input->gauges) + + cfl_list_size(&input->untypeds); + if (family_count > 0) { + rejected = flb_calloc(family_count, sizeof(*rejected)); + if (!rejected) { + flb_errno(); + goto failure; + } + } + + DETACH_UNSUPPORTED(counters, struct cmt_counter, counter, CMT_COUNTER); + DETACH_UNSUPPORTED(gauges, struct cmt_gauge, gauge, CMT_GAUGE); + DETACH_UNSUPPORTED(untypeds, struct cmt_untyped, untyped, CMT_UNTYPED); + + encoded = cmt_encode_opentelemetry_create(input); + + /* Restore the exact input list order before invoking any policy. */ + for (index = rejected_count; index > 0; index--) { + cfl_list_add(rejected[index - 1].head, rejected[index - 1].next); + } + + if (!encoded) { + goto failure; + } + + phase = "transform metric policies"; + ret = fts_telemetry_transform(ins, encoded, cfl_sds_len(encoded), &buffer, &size, + tag, tag_len); + cmt_encode_opentelemetry_destroy(encoded); + handled_errors = ret > 0; + if (ret < 0) { + goto failure; + } + + phase = "decode transformed metrics"; + ret = size ? cmt_decode_opentelemetry_create(&contexts, buffer, size, &offset) : 0; + flb_free(buffer); + buffer = NULL; + if (ret != 0) { + goto failure; + } + + phase = "merge native metric contexts"; + *output = merge_contexts(&contexts, input); + cmt_decode_opentelemetry_destroy(&contexts); + if (!*output) { + goto failure; + } + + if (rejected_count > 0) { + if (strcmp(ctx->error_mode, "keep") == 0) { + phase = "restore native uint64 family contexts"; + if (prepare_rejected_families(*output, input, rejected, rejected_count) != 0) { + cmt_destroy(*output); + *output = NULL; + goto failure; + } + + if (copy_rejected_families(*output, rejected, rejected_count) != 0) { + cmt_destroy(*output); + *output = NULL; + goto failure; + } + } + flb_plg_warn(ins, "FTS cannot process %zu native uint64 scalar families above " + "INT64_MAX; error_mode applied to these families", rejected_count); + } + + if (handled_errors || rejected_count > 0) { + fts_report_policy_error(ins); + } + + flb_free(rejected); + return FLB_PROCESSOR_SUCCESS; + +failure: + for (index = 0; index < rejected_count; index++) { + cfl_kvlist_destroy(rejected[index].metadata); + } + flb_free(rejected); + + fts_report_policy_error(ins); + flb_free(buffer); + cmt_decode_opentelemetry_destroy(&contexts); + flb_plg_error(ins, "cannot construct FTS metrics output (%s); edits discarded", phase); + + *output = strcmp(ctx->error_mode, "drop") == 0 ? cmt_create() : input; + return *output ? FLB_PROCESSOR_SUCCESS : FLB_PROCESSOR_FAILURE; +} diff --git a/plugins/processor_fts/fts_proto.c b/plugins/processor_fts/fts_proto.c new file mode 100644 index 00000000000..4bcd7c52344 --- /dev/null +++ b/plugins/processor_fts/fts_proto.c @@ -0,0 +1,622 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "fts_proto.h" + +/* Reflection is confined to the OTLP schema; policies see typed telemetry values. */ +static int reserve(struct fts_proto_budget *budget, size_t size) +{ + if (size > budget->limit || budget->used > budget->limit - size) { + return -1; + } + + budget->used += size; + return 0; +} + +void *fts_proto_alloc(void *data, size_t size) +{ + void *ptr; + + if (reserve(data, size) != 0) { + return NULL; + } + + ptr = calloc(1, size ? size : 1); + if (!ptr) { + flb_errno(); + } + + return ptr; +} + +void fts_proto_free(void *data, void *ptr) +{ + free(ptr); +} + +static size_t scalar_size(ProtobufCType type) +{ + switch (type) { + case PROTOBUF_C_TYPE_MESSAGE: + case PROTOBUF_C_TYPE_STRING: + return sizeof(void *); + case PROTOBUF_C_TYPE_BYTES: + return sizeof(ProtobufCBinaryData); + case PROTOBUF_C_TYPE_INT64: + case PROTOBUF_C_TYPE_UINT64: + case PROTOBUF_C_TYPE_FIXED64: + case PROTOBUF_C_TYPE_SFIXED64: + case PROTOBUF_C_TYPE_SINT64: + case PROTOBUF_C_TYPE_DOUBLE: + return 8; + default: + return 4; + } +} + +static int is_key_value(const ProtobufCFieldDescriptor *field) +{ + const ProtobufCMessageDescriptor *descriptor; + + if (field->type != PROTOBUF_C_TYPE_MESSAGE) { + return FLB_FALSE; + } + + descriptor = field->descriptor; + return strcmp(descriptor->short_name, "KeyValue") == 0; +} + +static struct cfl_variant *read_scalar(const ProtobufCFieldDescriptor *field, const void *ptr, + struct fts_proto_budget *budget, int depth) +{ + const ProtobufCBinaryData *bytes; + const char *string; + + if (reserve(budget, 256) != 0) { + return NULL; + } + + switch (field->type) { + case PROTOBUF_C_TYPE_MESSAGE: + return fts_proto_read(*(ProtobufCMessage *const *) ptr, budget, depth + 1); + case PROTOBUF_C_TYPE_STRING: + string = *(const char *const *) ptr; + string = string ? string : ""; + if (reserve(budget, strlen(string) + 1) != 0) { + return NULL; + } + return cfl_variant_create_from_string((char *) string); + case PROTOBUF_C_TYPE_BYTES: + bytes = ptr; + if (reserve(budget, bytes->len) != 0) { + return NULL; + } + return cfl_variant_create_from_bytes((char *) bytes->data, bytes->len, CFL_FALSE); + case PROTOBUF_C_TYPE_BOOL: + return cfl_variant_create_from_bool(*(const protobuf_c_boolean *) ptr); + case PROTOBUF_C_TYPE_ENUM: + case PROTOBUF_C_TYPE_INT32: + case PROTOBUF_C_TYPE_SINT32: + case PROTOBUF_C_TYPE_SFIXED32: + return cfl_variant_create_from_int64(*(const int32_t *) ptr); + case PROTOBUF_C_TYPE_UINT32: + case PROTOBUF_C_TYPE_FIXED32: + return cfl_variant_create_from_uint64(*(const uint32_t *) ptr); + case PROTOBUF_C_TYPE_INT64: + case PROTOBUF_C_TYPE_SINT64: + case PROTOBUF_C_TYPE_SFIXED64: + return cfl_variant_create_from_int64(*(const int64_t *) ptr); + case PROTOBUF_C_TYPE_UINT64: + case PROTOBUF_C_TYPE_FIXED64: + return cfl_variant_create_from_uint64(*(const uint64_t *) ptr); + case PROTOBUF_C_TYPE_DOUBLE: + return cfl_variant_create_from_double(*(const double *) ptr); + default: + return NULL; + } +} + +static struct cfl_variant *read_repeated(const ProtobufCFieldDescriptor *field, + const ProtobufCMessage *message, + struct fts_proto_budget *budget, int depth) +{ + size_t count; + size_t i; + const char *items; + struct cfl_variant *result; + struct cfl_variant *child; + struct cfl_kvlist *map; + struct cfl_array *array; + Opentelemetry__Proto__Common__V1__KeyValue *pair; + + count = *(const size_t *) ((const char *) message + field->quantifier_offset); + items = *(const char *const *) ((const char *) message + field->offset); + + if (count > (SIZE_MAX - 256) / 256 || reserve(budget, count * 256 + 256) != 0) { + return NULL; + } + + if (is_key_value(field)) { + map = cfl_kvlist_create(); + if (!map) { + return NULL; + } + result = cfl_variant_create_from_kvlist(map); + if (!result) { + cfl_kvlist_destroy(map); + return NULL; + } + } + else { + array = cfl_array_create(count); + if (!array) { + return NULL; + } + result = cfl_variant_create_from_array(array); + if (!result) { + cfl_array_destroy(array); + return NULL; + } + } + + for (i = 0; i < count; i++) { + if (is_key_value(field)) { + pair = ((Opentelemetry__Proto__Common__V1__KeyValue *const *) items)[i]; + if (!pair || !pair->key) { + goto failure; + } + child = fts_proto_read((ProtobufCMessage *) pair->value, budget, depth + 1); + if (!child || cfl_kvlist_insert(result->data.as_kvlist, pair->key, child) != 0) { + cfl_variant_destroy(child); + goto failure; + } + } + else { + child = read_scalar(field, items + i * scalar_size(field->type), budget, depth); + if (!child || cfl_array_append(result->data.as_array, child) != 0) { + cfl_variant_destroy(child); + goto failure; + } + } + } + + return result; + +failure: + cfl_variant_destroy(result); + return NULL; +} + +struct cfl_variant *fts_proto_read(const ProtobufCMessage *message, + struct fts_proto_budget *budget, int depth) +{ + const ProtobufCMessageDescriptor *descriptor; + const ProtobufCFieldDescriptor *field; + struct cfl_variant *result; + struct cfl_variant *child; + const char *ptr; + size_t i; + int any; + int wrapper; + + if (depth > FTS_MAX_DEPTH || reserve(budget, 256) != 0) { + return NULL; + } + + if (!message) { + return cfl_variant_create_from_null(); + } + + descriptor = message->descriptor; + any = strcmp(descriptor->short_name, "AnyValue") == 0; + wrapper = strcmp(descriptor->short_name, "ArrayValue") == 0 || + strcmp(descriptor->short_name, "KeyValueList") == 0; + + result = cfl_variant_create_from_kvlist(cfl_kvlist_create()); + if (!result) { + return NULL; + } + + for (i = 0; i < descriptor->n_fields; i++) { + field = &descriptor->fields[i]; + ptr = (const char *) message; + + if ((field->flags & PROTOBUF_C_FIELD_FLAG_ONEOF) && + *(const uint32_t *) (ptr + field->quantifier_offset) != field->id) { + continue; + } + + if (field->label == PROTOBUF_C_LABEL_OPTIONAL && field->quantifier_offset && + !(field->flags & PROTOBUF_C_FIELD_FLAG_ONEOF) && + !*(const protobuf_c_boolean *) (ptr + field->quantifier_offset)) { + continue; + } + + if (field->label != PROTOBUF_C_LABEL_REPEATED && field->type == PROTOBUF_C_TYPE_MESSAGE && + !*(const ProtobufCMessage *const *) (ptr + field->offset)) { + continue; + } + + if (field->label == PROTOBUF_C_LABEL_REPEATED) { + child = read_repeated(field, message, budget, depth); + } + else { + child = read_scalar(field, ptr + field->offset, budget, depth); + } + + if (!child) { + cfl_variant_destroy(result); + return NULL; + } + + if (any || wrapper) { + cfl_variant_destroy(result); + return child; + } + + if (reserve(budget, strlen(field->name) + 128) != 0 || + cfl_kvlist_insert(result->data.as_kvlist, (char *) field->name, child) != 0) { + cfl_variant_destroy(child); + cfl_variant_destroy(result); + return NULL; + } + } + + if (any) { + cfl_variant_destroy(result); + return cfl_variant_create_from_null(); + } + + return result; +} + +static int signed_integer(struct cfl_variant *value, int64_t *result) +{ + if (value->type == CFL_VARIANT_INT) { + *result = value->data.as_int64; + return 0; + } + + if (value->type == CFL_VARIANT_UINT && value->data.as_uint64 <= INT64_MAX) { + *result = (int64_t) value->data.as_uint64; + return 0; + } + + return -1; +} + +static int unsigned_integer(struct cfl_variant *value, uint64_t *result) +{ + if (value->type == CFL_VARIANT_UINT) { + *result = value->data.as_uint64; + return 0; + } + + if (value->type == CFL_VARIANT_INT && value->data.as_int64 >= 0) { + *result = (uint64_t) value->data.as_int64; + return 0; + } + + return -1; +} + +static int write_scalar(const ProtobufCFieldDescriptor *field, struct cfl_variant *value, + void *ptr, struct fts_proto_budget *budget, int depth) +{ + int64_t signed_value; + uint64_t unsigned_value; + size_t size; + char *string; + ProtobufCBinaryData *bytes; + + switch (field->type) { + case PROTOBUF_C_TYPE_MESSAGE: + *(ProtobufCMessage **) ptr = fts_proto_build(field->descriptor, value, budget, depth + 1); + return *(ProtobufCMessage **) ptr ? 0 : -1; + case PROTOBUF_C_TYPE_STRING: + if (value->type != CFL_VARIANT_STRING) { + return -1; + } + size = cfl_variant_size_get(value); + if (memchr(value->data.as_string, 0, size) || + !fts_valid_utf8(value->data.as_string, size) || reserve(budget, size + 1) != 0) { + return -1; + } + + string = malloc(size + 1); + if (!string) { + flb_errno(); + return -1; + } + + memcpy(string, value->data.as_string, size); + string[size] = 0; + *(char **) ptr = string; + return 0; + case PROTOBUF_C_TYPE_BYTES: + if (value->type != CFL_VARIANT_BYTES) { + return -1; + } + size = cfl_variant_size_get(value); + bytes = ptr; + bytes->data = fts_proto_alloc(budget, size); + if (!bytes->data) { + return -1; + } + + bytes->len = size; + memcpy(bytes->data, value->data.as_bytes, size); + return 0; + case PROTOBUF_C_TYPE_BOOL: + if (value->type != CFL_VARIANT_BOOL) { + return -1; + } + *(protobuf_c_boolean *) ptr = value->data.as_bool; + return 0; + case PROTOBUF_C_TYPE_ENUM: + case PROTOBUF_C_TYPE_INT32: + case PROTOBUF_C_TYPE_SINT32: + case PROTOBUF_C_TYPE_SFIXED32: + if (signed_integer(value, &signed_value) != 0 || + signed_value < INT32_MIN || signed_value > INT32_MAX) { + return -1; + } + *(int32_t *) ptr = (int32_t) signed_value; + return 0; + case PROTOBUF_C_TYPE_UINT32: + case PROTOBUF_C_TYPE_FIXED32: + if (unsigned_integer(value, &unsigned_value) != 0 || unsigned_value > UINT32_MAX) { + return -1; + } + *(uint32_t *) ptr = (uint32_t) unsigned_value; + return 0; + case PROTOBUF_C_TYPE_INT64: + case PROTOBUF_C_TYPE_SINT64: + case PROTOBUF_C_TYPE_SFIXED64: + return signed_integer(value, ptr); + case PROTOBUF_C_TYPE_UINT64: + case PROTOBUF_C_TYPE_FIXED64: + return unsigned_integer(value, ptr); + case PROTOBUF_C_TYPE_DOUBLE: + if (value->type == CFL_VARIANT_DOUBLE) { + *(double *) ptr = value->data.as_double; + } + else { + return -1; + } + return 0; + default: + return -1; + } +} + +static int write_repeated(const ProtobufCFieldDescriptor *field, struct cfl_variant *value, + ProtobufCMessage *message, struct fts_proto_budget *budget, int depth) +{ + size_t count; + size_t i; + size_t stride; + char *items; + struct cfl_list *head; + struct cfl_kvpair *pair; + Opentelemetry__Proto__Common__V1__KeyValue *entry; + ProtobufCMessage **messages; + + if (is_key_value(field)) { + if (value->type != CFL_VARIANT_KVLIST) { + return -1; + } + count = cfl_list_size(&value->data.as_kvlist->list); + } + else { + if (value->type != CFL_VARIANT_ARRAY) { + return -1; + } + count = value->data.as_array->entry_count; + } + + stride = scalar_size(field->type); + if (count > SIZE_MAX / stride) { + return -1; + } + + items = fts_proto_alloc(budget, count * stride); + if (!items) { + return -1; + } + + *(void **) ((char *) message + field->offset) = items; + *(size_t *) ((char *) message + field->quantifier_offset) = count; + + if (is_key_value(field)) { + i = 0; + messages = (ProtobufCMessage **) items; + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + entry = fts_proto_alloc(budget, sizeof(*entry)); + if (!entry) { + return -1; + } + + opentelemetry__proto__common__v1__key_value__init(entry); + messages[i++] = &entry->base; + + if (memchr(pair->key, 0, cfl_sds_len(pair->key)) || + !fts_valid_utf8(pair->key, cfl_sds_len(pair->key)) || + reserve(budget, cfl_sds_len(pair->key) + 1) != 0) { + return -1; + } + + /* + * protobuf-c releases the message with the system allocator, so + * the key is copied with malloc() like every other string. + */ + entry->key = malloc(cfl_sds_len(pair->key) + 1); + if (!entry->key) { + flb_errno(); + return -1; + } + memcpy(entry->key, pair->key, cfl_sds_len(pair->key) + 1); + + entry->value = (void *) fts_proto_build( + &opentelemetry__proto__common__v1__any_value__descriptor, + pair->val, budget, depth + 1); + + if (!entry->value) { + return -1; + } + } + } + else { + for (i = 0; i < count; i++) { + if (write_scalar(field, value->data.as_array->entries[i], items + i * stride, + budget, depth) != 0) { + return -1; + } + } + } + + return 0; +} + +ProtobufCMessage *fts_proto_build(const ProtobufCMessageDescriptor *descriptor, + struct cfl_variant *value, + struct fts_proto_budget *budget, int depth) +{ + ProtobufCMessage *message; + const ProtobufCFieldDescriptor *field; + struct cfl_list *head; + struct cfl_kvpair *pair; + int any; + int wrapper; + const char *name; + uint32_t *oneof; + + if (depth > FTS_MAX_DEPTH) { + return NULL; + } + + /* An adapter may retain a failed original item as opaque transport data. + * This path is private staging; policies never receive opaque messages. */ + if (value->type == CFL_VARIANT_BYTES && + (strcmp(descriptor->short_name, "ResourceMetrics") == 0 || + strcmp(descriptor->short_name, "ResourceSpans") == 0)) { + return protobuf_c_message_unpack(descriptor, NULL, cfl_variant_size_get(value), + (uint8_t *) value->data.as_bytes); + } + + message = fts_proto_alloc(budget, descriptor->sizeof_message); + if (!message) { + return NULL; + } + + descriptor->message_init(message); + any = strcmp(descriptor->short_name, "AnyValue") == 0; + wrapper = strcmp(descriptor->short_name, "ArrayValue") == 0 || + strcmp(descriptor->short_name, "KeyValueList") == 0; + + if (any) { + switch (value->type) { + case CFL_VARIANT_NULL: + return message; + case CFL_VARIANT_STRING: + name = "string_value"; + break; + case CFL_VARIANT_BYTES: + name = "bytes_value"; + break; + case CFL_VARIANT_BOOL: + name = "bool_value"; + break; + case CFL_VARIANT_INT: + case CFL_VARIANT_UINT: + name = "int_value"; + break; + case CFL_VARIANT_DOUBLE: + name = "double_value"; + break; + case CFL_VARIANT_ARRAY: + name = "array_value"; + break; + case CFL_VARIANT_KVLIST: + name = "kvlist_value"; + break; + default: + goto failure; + } + + field = protobuf_c_message_descriptor_get_field_by_name(descriptor, name); + *(uint32_t *) ((char *) message + field->quantifier_offset) = field->id; + + if (write_scalar(field, value, (char *) message + field->offset, budget, depth) != 0) { + goto failure; + } + return message; + } + + if (wrapper) { + if (write_repeated(&descriptor->fields[0], value, message, budget, depth) != 0) { + goto failure; + } + return message; + } + + if (value->type != CFL_VARIANT_KVLIST) { + goto failure; + } + + cfl_list_foreach(head, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + field = protobuf_c_message_descriptor_get_field_by_name(descriptor, pair->key); + if (!field || memchr(pair->key, 0, cfl_sds_len(pair->key)) || + fts_find_pair(value, pair->key, cfl_sds_len(pair->key)) != pair) { + goto failure; + } + + if (field->flags & PROTOBUF_C_FIELD_FLAG_ONEOF) { + oneof = (uint32_t *) ((char *) message + field->quantifier_offset); + if (*oneof != 0) { + goto failure; + } + *oneof = field->id; + } + else if (field->label == PROTOBUF_C_LABEL_OPTIONAL && field->quantifier_offset) { + *(protobuf_c_boolean *) ((char *) message + field->quantifier_offset) = 1; + } + + if (field->label == PROTOBUF_C_LABEL_REPEATED) { + if (write_repeated(field, pair->val, message, budget, depth) != 0) { + goto failure; + } + } + else if (write_scalar(field, pair->val, (char *) message + field->offset, + budget, depth) != 0) { + goto failure; + } + } + + return message; + +failure: + protobuf_c_message_free_unpacked(message, NULL); + return NULL; +} diff --git a/plugins/processor_fts/fts_proto.h b/plugins/processor_fts/fts_proto.h new file mode 100644 index 00000000000..19324881df4 --- /dev/null +++ b/plugins/processor_fts/fts_proto.h @@ -0,0 +1,39 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FTS_PROTO_H +#define FLB_FTS_PROTO_H + +#include "fts.h" +#include + +struct fts_proto_budget { + size_t used; + size_t limit; +}; + +void *fts_proto_alloc(void *data, size_t size); +void fts_proto_free(void *data, void *ptr); +struct cfl_variant *fts_proto_read(const ProtobufCMessage *message, + struct fts_proto_budget *budget, int depth); +ProtobufCMessage *fts_proto_build(const ProtobufCMessageDescriptor *descriptor, + struct cfl_variant *value, + struct fts_proto_budget *budget, int depth); + +#endif diff --git a/plugins/processor_fts/fts_runtime.c b/plugins/processor_fts/fts_runtime.c new file mode 100644 index 00000000000..68222f75b5e --- /dev/null +++ b/plugins/processor_fts/fts_runtime.c @@ -0,0 +1,475 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include + +#ifdef FLB_HAVE_SANITIZE_ADDRESS +#include +#endif + +#include "fts.h" + +/* + * LuaJIT raises errors by resetting the C stack pointer instead of calling + * longjmp(), so AddressSanitizer never sees the frames it skips and keeps + * their redzones poisoned. Later calls reusing that stack report bogus + * errors. After a failed protected call, clear the shadow of the stack below + * the current frame, which only holds those dead frames; the window covers + * nested C and Lua calls up to LuaJIT's C call limit. Builds without the + * sanitizer compile this to nothing. + */ +#define FTS_UNWOUND_STACK_SIZE (256 * 1024) + +#ifdef FLB_HAVE_SANITIZE_ADDRESS +__attribute__((noinline)) +#endif +void fts_sanitizer_forget_unwound_frames(void) +{ +#ifdef FLB_HAVE_SANITIZE_ADDRESS + char *top; + + top = (char *) __builtin_frame_address(0); + __asan_unpoison_memory_region(top - FTS_UNWOUND_STACK_SIZE, FTS_UNWOUND_STACK_SIZE); +#endif +} + +/* A policy cannot catch budget exhaustion and continue an endless loop. */ +static int policy_pcall(lua_State *l) +{ + struct fts_runtime *ctx; + void *data; + int ret; + + luaL_checktype(l, 1, LUA_TFUNCTION); + ret = lua_pcall(l, lua_gettop(l) - 1, LUA_MULTRET, 0); + if (ret != 0) { + fts_sanitizer_forget_unwound_frames(); + } + + lua_getallocf(l, &data); + ctx = data; + if (!ctx->jit && ctx->instructions_left <= 0) { + return luaL_error(l, "Lua instruction limit exceeded"); + } + + lua_pushboolean(l, ret == 0); + lua_insert(l, 1); + return lua_gettop(l); +} + +/* Enforce the count budget in interpreted Lua, including script initialization. */ +static void budget_hook(lua_State *l, lua_Debug *ar) +{ + struct fts_runtime *ctx; + void *data; + + lua_getallocf(l, &data); + ctx = data; + ctx->instructions_left -= 1000; + + if (!ctx->jit && ctx->instructions_left <= 0) { + luaL_error(l, "Lua instruction limit exceeded"); + } +} + +static void *bounded_allocator(void *data, void *ptr, size_t old_size, size_t new_size) +{ + struct fts_runtime *ctx = data; + void *result; + size_t used; + + if (!ptr) { + old_size = 0; + } + + if (ctx->lua_bytes >= old_size) { + used = ctx->lua_bytes - old_size; + } + else { + used = 0; + } + + if (new_size > old_size && + (new_size > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - new_size || + used > ctx->memory_limit - ctx->native_bytes - new_size)) { + return NULL; + } + + result = ctx->original_allocator(ctx->allocator_data, ptr, old_size, new_size); + if (result || new_size == 0) { + ctx->lua_bytes = used + new_size; + } + + return result; +} + +static int policy_collectgarbage(lua_State *l) +{ + const char *option; + int arguments; + + option = luaL_optstring(l, 1, "collect"); + if (strcmp(option, "collect") != 0 && strcmp(option, "count") != 0 && + strcmp(option, "step") != 0) { + return luaL_error(l, "FTS collectgarbage supports collect, count and step only"); + } + + arguments = lua_gettop(l); + lua_pushvalue(l, lua_upvalueindex(1)); + lua_insert(l, 1); + lua_call(l, arguments, LUA_MULTRET); + return lua_gettop(l); +} + +static int restricted_require(lua_State *l) +{ + const char *name; + size_t length; + + name = luaL_checklstring(l, 1, &length); + if (length != sizeof("fluentbit") - 1 || memcmp(name, "fluentbit", length) != 0) { + return luaL_error(l, "only the fluentbit module is available in FTS policies"); + } + + lua_pushvalue(l, lua_upvalueindex(1)); + return 1; +} + +void fts_runtime_destroy(struct fts_runtime *ctx) +{ + fts_undo_commit(ctx); + flb_free(ctx->undo); + ctx->undo = NULL; + ctx->undo_size = 0; + + if (ctx->vm) { + flb_luajit_destroy(ctx->vm); + ctx->vm = NULL; + } +} + +/* + * Helper code has a private copy of the standard environment. Policies cannot + * replace validation primitives or discover this environment through getfenv(). + */ +static void copy_table(lua_State *l, int index) +{ + int source; + int target; + + if (index < 0 && index > LUA_REGISTRYINDEX) { + source = lua_gettop(l) + index + 1; + } + else { + source = index; + } + + lua_newtable(l); + target = lua_gettop(l); + + lua_pushnil(l); + while (lua_next(l, source)) { + lua_pushvalue(l, -2); + lua_pushvalue(l, -2); + lua_rawset(l, target); + lua_pop(l, 1); + } +} + +int fts_load_helpers(lua_State *l, const char *code, size_t size, const char *name) +{ + struct fts_runtime *ctx; + void *data; + + lua_getallocf(l, &data); + ctx = data; + + if (luaL_loadbuffer(l, code, size, name) != 0) { + return -1; + } + + lua_rawgeti(l, LUA_REGISTRYINDEX, ctx->helpers_env_ref); + lua_setfenv(l, -2); + if (lua_pcall(l, 0, 0, 0) != 0) { + fts_sanitizer_forget_unwound_frames(); + return -1; + } + return 0; +} + +static int initialize_policy(lua_State *l) +{ + struct fts_runtime *ctx; + const char *blocked[] = {"io", "os", "debug", "jit", "ffi", "package", "coroutine", + "module", "load", "loadfile", "loadstring", "dofile", + "newproxy", "xpcall", "getfenv", "setfenv", "print", NULL}; + int i; + int ret; + int environment; + + ctx = lua_touserdata(l, 1); + copy_table(l, LUA_GLOBALSINDEX); + environment = lua_gettop(l); + + /* tables used by validation must also be independent of policy globals */ + lua_getglobal(l, "math"); + copy_table(l, -1); + lua_setfield(l, environment, "math"); + lua_pop(l, 1); + + lua_getglobal(l, "string"); + copy_table(l, -1); + lua_setglobal(l, "string"); + lua_pop(l, 1); + + lua_getglobal(l, "table"); + copy_table(l, -1); + lua_setfield(l, environment, "table"); + lua_pop(l, 1); + + lua_pushliteral(l, ""); + lua_getmetatable(l, -1); + lua_pushliteral(l, "string"); + lua_setfield(l, -2, "__metatable"); + lua_pop(l, 2); + + ctx->helpers_env_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + if (fts_register_values(l) != 0) { + return luaL_error(l, "cannot register fluentbit module"); + } + + if (ctx->adapter->register_helpers(l) != 0) { + return luaL_error(l, "cannot register FTS signal helpers"); + } + + lua_getglobal(l, "package"); + lua_getfield(l, -1, "loaded"); + lua_getfield(l, -1, "fluentbit"); + lua_pushvalue(l, -1); + ctx->module_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + /* public helper overrides do not affect the private adapter module */ + copy_table(l, -1); + lua_pushcclosure(l, restricted_require, 1); + lua_setglobal(l, "require"); + + /* removing globals alone leaves Lua 5.1 module() able to recover loaded libraries */ + for (i = 0; blocked[i]; i++) { + lua_pushnil(l); + lua_setfield(l, -2, blocked[i]); + } + lua_pop(l, 3); + + for (i = 0; blocked[i]; i++) { + lua_pushnil(l); + lua_setglobal(l, blocked[i]); + } + + lua_pushcfunction(l, policy_pcall); + lua_setglobal(l, "pcall"); + + lua_getglobal(l, "collectgarbage"); + lua_pushcclosure(l, policy_collectgarbage, 1); + lua_setglobal(l, "collectgarbage"); + + if (ctx->code) { + ret = flb_luajit_load_buffer(ctx->vm, ctx->code, strlen(ctx->code), "fts_policy"); + } + else { + ret = flb_luajit_load_script(ctx->vm, ctx->script); + } + + if (ret != 0) { + return lua_error(l); + } + + lua_call(l, 0, 0); + + lua_getglobal(l, ctx->call); + if (!lua_isfunction(l, -1)) { + return luaL_error(l, "FTS callback function not found"); + } + ctx->policy_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + /* field-name strings are callback upvalues, see fts_getfield() */ + for (i = 0; i < FTS_KEY_COUNT; i++) { + lua_rawgeti(l, LUA_REGISTRYINDEX, ctx->key_refs[i]); + } + lua_pushcclosure(l, ctx->adapter->run_callback, FTS_KEY_COUNT); + ctx->callback_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + if (!lua_checkstack(l, 32)) { + return luaL_error(l, "cannot reserve policy stack"); + } + + return 0; +} + +int fts_runtime_initialize(struct fts_runtime *ctx, struct flb_config *config) +{ + lua_State *l; + int ret; + + ctx->vm = flb_luajit_create(config); + if (!ctx->vm) { + return FLB_PROCESSOR_FAILURE; + } + l = ctx->vm->state; + + ret = luaJIT_setmode(l, 0, LUAJIT_MODE_ENGINE | (ctx->jit ? LUAJIT_MODE_ON : LUAJIT_MODE_OFF)); + if (ctx->jit && !ret) { + lua_pushliteral(l, "JIT is unavailable in this LuaJIT build"); + return FLB_PROCESSOR_FAILURE; + } + + ctx->original_allocator = lua_getallocf(l, &ctx->allocator_data); + ctx->lua_bytes = (size_t) lua_gc(l, LUA_GCCOUNT, 0) * 1024 + lua_gc(l, LUA_GCCOUNTB, 0); + lua_setallocf(l, bounded_allocator, ctx); + + ctx->instructions_left = ctx->instruction_limit; + if (!ctx->jit) { + lua_sethook(l, budget_hook, LUA_MASKCOUNT, 1000); + } + + lua_pushcfunction(l, initialize_policy); + lua_pushlightuserdata(l, ctx); + ret = lua_pcall(l, 1, 0, 0); + if (ret != 0) { + fts_sanitizer_forget_unwound_frames(); + return FLB_PROCESSOR_FAILURE; + } + + return FLB_PROCESSOR_SUCCESS; +} + +/* + * Native values are invisible to the Lua allocator, so the collector is paced + * explicitly: each chunk performs incremental work proportional to the native + * bytes it charged. A full collection only runs on emergency or when close to + * the memory limit, avoiding a full cycle per chunk or error. + */ +void fts_collect(struct fts_runtime *ctx, int emergency) +{ + lua_State *l = ctx->vm->state; + size_t used = ctx->native_bytes + ctx->lua_bytes; + size_t allocated; + + allocated = ctx->native_allocated; + ctx->native_allocated = 0; + + if (emergency || used > ctx->memory_limit - ctx->memory_limit / 4) { + lua_gc(l, LUA_GCCOLLECT, 0); + } + else if (allocated > 0) { + lua_gc(l, LUA_GCSTEP, (int) (allocated / 1024 < INT_MAX ? allocated / 1024 + 1 : INT_MAX)); + } + else if (used > ctx->memory_limit / 2) { + lua_gc(l, LUA_GCSTEP, 64); + } +} + +/* adapters supply their own snapshot and result contract inside the protected call */ +int fts_runtime_call(struct fts_runtime *ctx, void *data, int results) +{ + lua_State *l; + int ret; + int base; + + l = ctx->vm->state; + base = lua_gettop(l); + ctx->instructions_left = ctx->instruction_limit; + + lua_rawgeti(l, LUA_REGISTRYINDEX, ctx->callback_ref); + lua_pushlightuserdata(l, data); + ret = lua_pcall(l, 1, results, 0); + if (ret != 0) { + fts_sanitizer_forget_unwound_frames(); + } + if (ret == 0 && !ctx->jit && ctx->instructions_left <= 0) { + lua_settop(l, base); + lua_pushliteral(l, "Lua instruction limit exceeded"); + ret = LUA_ERRRUN; + } + + if (ret != 0) { + /* unwound snapshots and garbage must not poison subsequent callbacks */ + fts_collect(ctx, ret == LUA_ERRMEM); + if (!lua_gc(l, LUA_GCISRUNNING, 0)) { + lua_gc(l, LUA_GCRESTART, -1); + } + } + + return ret; +} + +void fts_report_policy_error(struct flb_processor_instance *ins) +{ + struct fts_runtime *ctx; + struct flb_processor *processor; + struct flb_input_instance *input; + struct flb_output_instance *output; + char stage[32]; + char *labels[5]; + + if (!ins->pu || !ins->pu->parent) { + return; + } + + processor = ins->pu->parent; + if (!processor->cmt_errors) { + return; + } + + labels[0] = "unknown"; + labels[1] = "unknown"; + if (processor->source_plugin_type == FLB_PLUGIN_INPUT && processor->data) { + input = processor->data; + labels[0] = "input"; + labels[1] = (char *) flb_input_name(input); + } + else if (processor->source_plugin_type == FLB_PLUGIN_OUTPUT && processor->data) { + output = processor->data; + labels[0] = "output"; + labels[1] = (char *) flb_output_name(output); + } + + snprintf(stage, sizeof(stage), "%zu", ins->pu->stage); + labels[2] = ins->pu->name; + labels[3] = stage; + + ctx = ins->context; + if (ctx->adapter->event_type == FLB_PROCESSOR_METRICS) { + labels[4] = "metrics"; + } + else if (ctx->adapter->event_type == FLB_PROCESSOR_TRACES) { + labels[4] = "traces"; + } + else { + labels[4] = "logs"; + } + + cmt_counter_inc(processor->cmt_errors, cfl_time_now(), 5, labels); +} + diff --git a/plugins/processor_fts/fts_telemetry.c b/plugins/processor_fts/fts_telemetry.c new file mode 100644 index 00000000000..dfa9a1ba6bd --- /dev/null +++ b/plugins/processor_fts/fts_telemetry.c @@ -0,0 +1,815 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include "fts_telemetry.h" +#include "fts_helpers.h" + +struct telemetry_call { + struct fts_runtime *ctx; + struct cfl_variant *snapshot; + const char *tag; + int tag_len; +}; + +static struct cfl_variant *field(struct cfl_variant *map, const char *key) +{ + struct cfl_kvpair *pair; + + if (!map || map->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + pair = fts_find_pair(map, key, strlen(key)); + if (!pair) { + return NULL; + } + + return pair->val; +} + +static struct cfl_variant *new_map(void) +{ + struct cfl_kvlist *map; + struct cfl_variant *value; + + map = cfl_kvlist_create(); + if (!map) { + return NULL; + } + + value = cfl_variant_create_from_kvlist(map); + if (!value) { + cfl_kvlist_destroy(map); + } + + return value; +} + +static struct cfl_variant *new_array(void) +{ + struct cfl_array *array; + struct cfl_variant *value; + + array = cfl_array_create(0); + if (!array) { + return NULL; + } + cfl_array_resizable(array, CFL_TRUE); + + value = cfl_variant_create_from_array(array); + if (!value) { + cfl_array_destroy(array); + } + + return value; +} + +/* the receiver owns value on success and failure */ +static int put(struct cfl_variant *map, const char *key, struct cfl_variant *value) +{ + if (!value || cfl_kvlist_insert(map->data.as_kvlist, (char *) key, value) != 0) { + cfl_variant_destroy(value); + return -1; + } + + return 0; +} + +static int clone_field(struct cfl_variant *map, const char *name, struct cfl_variant *source) +{ + struct cfl_variant *value; + + if (source) { + value = fts_copy_value(source, 0); + } + else { + value = new_map(); + } + + return put(map, name, value); +} + +static struct cfl_variant *snapshot(struct cfl_variant *resource, struct cfl_variant *scope, + struct cfl_variant *item) +{ + struct cfl_variant *result; + struct cfl_variant *context; + struct cfl_variant *url; + struct cfl_variant *value; + + result = new_map(); + if (!result) { + return NULL; + } + + if (clone_field(result, "item", item) != 0 || + clone_field(result, "resource", field(resource, "resource")) != 0 || + clone_field(result, "scope", field(scope, "scope")) != 0) { + goto failure; + } + + context = field(result, "resource"); + if (!field(context, "attributes") && put(context, "attributes", new_map()) != 0) { + goto failure; + } + + context = field(result, "scope"); + if (!field(context, "attributes") && put(context, "attributes", new_map()) != 0) { + goto failure; + } + + url = field(resource, "schema_url"); + if (url) { + value = fts_copy_value(url, 0); + } + else { + value = cfl_variant_create_from_string(""); + } + if (put(result, "resource_schema_url", value) != 0) { + goto failure; + } + + url = field(scope, "schema_url"); + if (url) { + value = fts_copy_value(url, 0); + } + else { + value = cfl_variant_create_from_string(""); + } + if (put(result, "scope_schema_url", value) != 0) { + goto failure; + } + + return result; + +failure: + cfl_variant_destroy(result); + return NULL; +} + +int fts_register_telemetry_helpers(lua_State *l) +{ + if (fts_load_helpers(l, fts_telemetry_helpers, sizeof(fts_telemetry_helpers) - 1, + "fluentbit.telemetry_helpers") != 0) { + return -1; + } + + return 0; +} + +int fts_run_telemetry(lua_State *l) +{ + struct telemetry_call *call; + struct fts_value *owner; + struct fts_value *emissions; + const char *signal; + const char *result; + size_t length; + size_t i; + int original; + int raw; + int output; + int module; + int view; + int drop = FLB_FALSE; + + call = lua_touserdata(l, 1); + if (call->ctx->adapter->event_type == FLB_PROCESSOR_METRICS) { + signal = "metrics"; + } + else { + signal = "traces"; + } + + /* mutable snapshot handed to the policy */ + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, fts_value_cost(call->snapshot, 0)); + owner->owned = fts_copy_value(call->snapshot, 0); + if (!owner->owned) { + return luaL_error(l, "cannot create telemetry snapshot"); + } + raw = lua_gettop(l); + + /* read-only copy used to validate the edits */ + owner = fts_push_owned(l, NULL, FLB_TRUE); + fts_charge_value(l, owner, fts_value_cost(call->snapshot, 0)); + owner->owned = fts_copy_value(call->snapshot, 0); + if (!owner->owned) { + return luaL_error(l, "cannot preserve original telemetry fields"); + } + original = lua_gettop(l); + + emissions = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, emissions, 256); + emissions->owned = new_array(); + if (!emissions->owned) { + return luaL_error(l, "cannot create emissions array"); + } + output = lua_gettop(l); + + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->module_ref); + module = lua_gettop(l); + + lua_getfield(l, module, "_fts_wrap"); + lua_pushvalue(l, raw); + lua_pushstring(l, signal); + lua_pushvalue(l, output); + if (call->tag) { + lua_pushlstring(l, call->tag, call->tag_len); + } + else { + lua_pushlstring(l, "", 0); + } + lua_call(l, 4, 1); + view = lua_gettop(l); + + lua_rawgeti(l, LUA_REGISTRYINDEX, call->ctx->policy_ref); + lua_pushvalue(l, view); + lua_call(l, 1, 1); + + if (!lua_isnil(l, -1)) { + result = NULL; + if (lua_type(l, -1) == LUA_TSTRING) { + result = lua_tolstring(l, -1, &length); + } + + if (!result || length != 4 || (memcmp(result, "keep", 4) && memcmp(result, "drop", 4))) { + return luaL_error(l, "callback must return nil, fb.KEEP or fb.DROP"); + } + drop = memcmp(result, "drop", 4) == 0; + } + + if (!drop) { + lua_getfield(l, module, "_fts_validate_telemetry"); + lua_pushvalue(l, raw); + lua_pushstring(l, signal); + lua_pushvalue(l, original); + lua_call(l, 3, 0); + } + + for (i = 0; i < emissions->owned->data.as_array->entry_count; i++) { + lua_getfield(l, module, "_fts_validate_telemetry"); + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, fts_value_cost(emissions->owned->data.as_array->entries[i], 0)); + owner->owned = fts_copy_value(emissions->owned->data.as_array->entries[i], 0); + if (!owner->owned) { + return luaL_error(l, "cannot validate emitted metric"); + } + lua_pushstring(l, signal); + lua_call(l, 2, 0); + } + + lua_pushstring(l, drop ? "drop" : "keep"); + lua_pushvalue(l, raw); + lua_pushvalue(l, output); + return 3; +} + +static struct cfl_variant *group_from_snapshot(struct cfl_variant *value, int metrics) +{ + struct cfl_variant *resource; + struct cfl_variant *scope; + struct cfl_variant *scopes; + struct cfl_variant *items; + struct cfl_variant *item = NULL; + + resource = new_map(); + scope = new_map(); + scopes = new_array(); + items = new_array(); + if (!resource || !scope || !scopes || !items) { + goto failure; + } + + if (clone_field(resource, "resource", field(value, "resource")) != 0 || + clone_field(resource, "schema_url", field(value, "resource_schema_url")) != 0 || + clone_field(scope, "scope", field(value, "scope")) != 0 || + clone_field(scope, "schema_url", field(value, "scope_schema_url")) != 0) { + goto failure; + } + + item = fts_copy_value(field(value, "item"), 0); + if (!item || cfl_array_append(items->data.as_array, item) != 0) { + goto failure; + } + item = NULL; + + if (put(scope, metrics ? "metrics" : "spans", items) != 0) { + items = NULL; + goto failure; + } + items = NULL; + + if (cfl_array_append(scopes->data.as_array, scope) != 0) { + goto failure; + } + scope = NULL; + + if (put(resource, metrics ? "scope_metrics" : "scope_spans", scopes) != 0) { + scopes = NULL; + goto failure; + } + + return resource; + +failure: + cfl_variant_destroy(resource); + cfl_variant_destroy(scope); + cfl_variant_destroy(scopes); + cfl_variant_destroy(items); + cfl_variant_destroy(item); + return NULL; +} + +static int stage(struct cfl_variant *groups, struct cfl_variant *value, int metrics, + size_t *used, size_t limit) +{ + struct cfl_variant *group; + ProtobufCMessage *message; + struct fts_proto_budget budget = {0, limit}; + const ProtobufCMessageDescriptor *descriptor; + ProtobufCMessage *item; + size_t cost; + size_t length; + size_t prefix; + + group = group_from_snapshot(value, metrics); + if (!group) { + return -1; + } + + if (metrics) { + descriptor = &opentelemetry__proto__metrics__v1__resource_metrics__descriptor; + } + else { + descriptor = &opentelemetry__proto__trace__v1__resource_spans__descriptor; + } + + message = fts_proto_build(descriptor, group, &budget, 0); + if (!message) { + cfl_variant_destroy(group); + return -1; + } + + if (metrics) { + item = &((Opentelemetry__Proto__Metrics__V1__ResourceMetrics *) message)-> + scope_metrics[0]->metrics[0]->base; + } + else { + item = &((Opentelemetry__Proto__Trace__V1__ResourceSpans *) message)-> + scope_spans[0]->spans[0]->base; + } + + /* + * Account item bytes and its repeated-field framing here. Coalesced + * resource/scope bytes are checked once in the final serialized output. + */ + cost = protobuf_c_message_get_packed_size(item); + prefix = 1; + for (length = cost; length >= 128; length >>= 7) { + prefix++; + } + cost += prefix + 1; + protobuf_c_message_free_unpacked(message, NULL); + + if (cost > limit || *used > limit - cost || + cfl_array_append(groups->data.as_array, group) != 0) { + cfl_variant_destroy(group); + return -1; + } + + *used += cost; + return 0; +} + +/* Coalesce adjacent unchanged contexts without teaching policies about groups. */ +static int append_group(struct cfl_variant *groups, struct cfl_variant *group, int metrics) +{ + struct cfl_array *list; + struct cfl_variant *previous; + struct cfl_variant *a; + struct cfl_variant *b; + struct cfl_variant *items; + struct cfl_variant *source; + struct cfl_variant *item; + const char *scope_key = metrics ? "scope_metrics" : "scope_spans"; + const char *item_key = metrics ? "metrics" : "spans"; + size_t i; + + list = groups->data.as_array; + if (group->type != CFL_VARIANT_BYTES && list->entry_count > 0 && + list->entries[list->entry_count - 1]->type != CFL_VARIANT_BYTES) { + previous = list->entries[list->entry_count - 1]; + a = field(previous, scope_key)->data.as_array->entries[0]; + b = field(group, scope_key)->data.as_array->entries[0]; + + if (fts_values_equal(field(previous, "resource"), field(group, "resource"), 0) && + fts_values_equal(field(previous, "schema_url"), field(group, "schema_url"), 0) && + fts_values_equal(field(a, "scope"), field(b, "scope"), 0) && + fts_values_equal(field(a, "schema_url"), field(b, "schema_url"), 0)) { + items = field(a, item_key); + source = field(b, item_key); + for (i = 0; i < source->data.as_array->entry_count; i++) { + item = source->data.as_array->entries[i]; + source->data.as_array->entries[i] = NULL; + cfl_container_release_variant(item); + if (cfl_array_append(items->data.as_array, item) != 0) { + cfl_variant_destroy(item); + cfl_variant_destroy(group); + return -1; + } + } + cfl_variant_destroy(group); + return 0; + } + } + + if (cfl_array_append(groups->data.as_array, group) != 0) { + cfl_variant_destroy(group); + return -1; + } + + return 0; +} + +/* Decode resource/scope context without recursively materializing their items. */ +static struct cfl_variant *read_context(const ProtobufCMessage *message, + const char *children, struct fts_proto_budget *budget) +{ + const ProtobufCFieldDescriptor *descriptor; + ProtobufCMessage *copy; + struct cfl_variant *result; + + descriptor = protobuf_c_message_descriptor_get_field_by_name(message->descriptor, children); + copy = flb_malloc(message->descriptor->sizeof_message); + if (!copy) { + flb_errno(); + return NULL; + } + + if (!descriptor) { + flb_free(copy); + return NULL; + } + + /* shallow copy with the children field emptied */ + memcpy(copy, message, message->descriptor->sizeof_message); + *(size_t *) ((char *) copy + descriptor->quantifier_offset) = 0; + *(void **) ((char *) copy + descriptor->offset) = NULL; + + budget->used = 0; + result = fts_proto_read(copy, budget, 0); + flb_free(copy); + + return result; +} + +static ProtobufCMessage **children_of(const ProtobufCMessage *message, + const char *name, size_t *count) +{ + const ProtobufCFieldDescriptor *field; + + field = protobuf_c_message_descriptor_get_field_by_name(message->descriptor, name); + *count = *(size_t *) ((char *) message + field->quantifier_offset); + return *(ProtobufCMessage ***) ((char *) message + field->offset); +} + +/* + * Keep a failed original item without requiring it to fit the typed API's + * nesting/memory budget. Its siblings still receive independent callbacks. + */ +static struct cfl_variant *opaque_group(ProtobufCMessage *resource, + ProtobufCMessage *scope, ProtobufCMessage *item, + int metrics) +{ + const ProtobufCFieldDescriptor *resource_field; + const ProtobufCFieldDescriptor *scope_field; + const char *scope_name; + const char *item_name; + ProtobufCMessage *resource_copy; + ProtobufCMessage *scope_copy; + struct cfl_variant *result; + uint8_t *buffer; + size_t size; + + resource_copy = flb_malloc(resource->descriptor->sizeof_message); + scope_copy = flb_malloc(scope->descriptor->sizeof_message); + if (!resource_copy || !scope_copy) { + flb_errno(); + flb_free(resource_copy); + flb_free(scope_copy); + return NULL; + } + + /* shallow copies linked so they only reference the single item */ + memcpy(resource_copy, resource, resource->descriptor->sizeof_message); + memcpy(scope_copy, scope, scope->descriptor->sizeof_message); + + if (metrics) { + scope_name = "scope_metrics"; + item_name = "metrics"; + } + else { + scope_name = "scope_spans"; + item_name = "spans"; + } + + resource_field = protobuf_c_message_descriptor_get_field_by_name(resource->descriptor, + scope_name); + scope_field = protobuf_c_message_descriptor_get_field_by_name(scope->descriptor, item_name); + + *(size_t *) ((char *) resource_copy + resource_field->quantifier_offset) = 1; + *(ProtobufCMessage ***) ((char *) resource_copy + resource_field->offset) = &scope_copy; + *(size_t *) ((char *) scope_copy + scope_field->quantifier_offset) = 1; + *(ProtobufCMessage ***) ((char *) scope_copy + scope_field->offset) = &item; + + size = protobuf_c_message_get_packed_size(resource_copy); + buffer = flb_malloc(size ? size : 1); + if (!buffer) { + flb_errno(); + flb_free(scope_copy); + flb_free(resource_copy); + return NULL; + } + + protobuf_c_message_pack(resource_copy, buffer); + result = cfl_variant_create_from_bytes((char *) buffer, size, CFL_FALSE); + flb_free(buffer); + + flb_free(scope_copy); + flb_free(resource_copy); + + return result; +} + +int fts_telemetry_transform(struct flb_processor_instance *ins, char *buffer, size_t size, + char **output, size_t *output_size, const char *tag, int tag_len) +{ + struct fts_runtime *ctx = ins->context; + struct fts_proto_budget budget; + ProtobufCMessage *message = NULL; + const ProtobufCMessageDescriptor *descriptor; + struct cfl_variant *resource = NULL; + struct cfl_variant *scope = NULL; + struct cfl_variant *original_item = NULL; + struct cfl_variant *result = NULL; + struct cfl_variant *groups = NULL; + ProtobufCMessage **resources; + ProtobufCMessage **scopes; + ProtobufCMessage **items; + size_t resource_count; + size_t scope_count; + size_t item_count; + struct cfl_variant *value = NULL; + struct cfl_variant *pending = NULL; + struct cfl_variant *item; + struct fts_value *edited; + struct fts_value *emissions; + struct telemetry_call call; + lua_State *l = ctx->vm->state; + size_t r; + size_t s; + size_t i; + size_t e; + size_t used = 0; + size_t pending_cost; + int metrics = ctx->adapter->event_type == FLB_PROCESSOR_METRICS; + const char *resource_key = metrics ? "resource_metrics" : "resource_spans"; + const char *scope_key = metrics ? "scope_metrics" : "scope_spans"; + const char *item_key = metrics ? "metrics" : "spans"; + int base; + int ret; + int errors = 0; + int failed; + const char *phase = "decode OTLP"; + + *output = NULL; + *output_size = 0; + + if (metrics) { + descriptor = &opentelemetry__proto__metrics__v1__metrics_data__descriptor; + } + else { + descriptor = + &opentelemetry__proto__collector__trace__v1__export_trace_service_request__descriptor; + } + + budget.used = 0; + budget.limit = ctx->memory_limit; + + message = protobuf_c_message_unpack(descriptor, NULL, size, (uint8_t *) buffer); + if (!message) { + goto failure; + } + + result = new_map(); + groups = new_array(); + if (!result || !groups) { + goto failure; + } + + phase = "read resource groups"; + resources = children_of(message, resource_key, &resource_count); + for (r = 0; r < resource_count; r++) { + resource = read_context(resources[r], scope_key, &budget); + scopes = children_of(resources[r], scope_key, &scope_count); + for (s = 0; s < scope_count; s++) { + scope = read_context(scopes[s], item_key, &budget); + items = children_of(scopes[s], item_key, &item_count); + for (i = 0; i < item_count; i++) { + phase = "create telemetry snapshot"; + budget.used = 0; + + original_item = NULL; + if (resource && scope) { + original_item = fts_proto_read(items[i], &budget, 0); + } + + value = NULL; + if (original_item) { + value = snapshot(resource, scope, original_item); + } + cfl_variant_destroy(original_item); + original_item = NULL; + + if (!value) { + errors++; + if (errors <= 5) { + flb_plg_error(ins, "FTS telemetry item exceeds snapshot limits; %s", + strcmp(ctx->error_mode, "keep") == 0 ? + "original item kept" : "item dropped"); + } + + if (strcmp(ctx->error_mode, "keep") == 0) { + item = opaque_group(resources[r], scopes[s], items[i], metrics); + if (!item || append_group(groups, item, metrics) != 0) { + goto failure; + } + } + continue; + } + + base = lua_gettop(l); + call.ctx = ctx; + call.snapshot = value; + call.tag = tag; + call.tag_len = tag_len; + ret = fts_runtime_call(ctx, &call, 3); + + pending = new_array(); + pending_cost = used; + failed = ret != 0 || !pending; + + if (!failed) { + edited = lua_touserdata(l, base + 2); + emissions = lua_touserdata(l, base + 3); + + if (strcmp(lua_tostring(l, base + 1), "keep") == 0 && + stage(pending, edited->owned, metrics, &pending_cost, + ctx->output_limit) != 0) { + failed = FLB_TRUE; + } + + for (e = 0; !failed && e < emissions->owned->data.as_array->entry_count; e++) { + if (stage(pending, emissions->owned->data.as_array->entries[e], metrics, + &pending_cost, ctx->output_limit) != 0) { + failed = FLB_TRUE; + } + } + } + + if (failed) { + errors++; + if (errors <= 5) { + flb_plg_error(ins, "FTS telemetry policy failed: %.256s; edits rolled back", + ret && lua_type(l, -1) == LUA_TSTRING ? + lua_tostring(l, -1) : + "invalid telemetry fields or output limit"); + } + + /* restart staging, keeping the original item when configured */ + cfl_variant_destroy(pending); + pending = new_array(); + pending_cost = used; + if (!pending || (strcmp(ctx->error_mode, "keep") == 0 && + stage(pending, value, metrics, &pending_cost, + ctx->output_limit) != 0)) { + lua_settop(l, base); + goto failure; + } + } + + for (e = 0; e < pending->data.as_array->entry_count; e++) { + item = pending->data.as_array->entries[e]; + pending->data.as_array->entries[e] = NULL; + cfl_container_release_variant(item); + if (append_group(groups, item, metrics) != 0) { + lua_settop(l, base); + goto failure; + } + } + + used = pending_cost; + cfl_variant_destroy(pending); + pending = NULL; + cfl_variant_destroy(value); + value = NULL; + lua_settop(l, base); + } + cfl_variant_destroy(scope); + scope = NULL; + } + cfl_variant_destroy(resource); + resource = NULL; + } + + if (put(result, resource_key, groups) != 0) { + groups = NULL; + goto failure; + } + groups = NULL; + + protobuf_c_message_free_unpacked(message, NULL); + message = NULL; + + /* transport staging is bounded by output_limit, independently of the VM */ + budget.used = 0; + budget.limit = SIZE_MAX; + phase = "encode typed OTLP values"; + + message = fts_proto_build(descriptor, result, &budget, 0); + if (!message) { + goto failure; + } + + *output_size = protobuf_c_message_get_packed_size(message); + if (*output_size > ctx->output_limit) { + goto failure; + } + + *output = flb_malloc(*output_size ? *output_size : 1); + if (!*output) { + flb_errno(); + goto failure; + } + + protobuf_c_message_pack(message, (uint8_t *) *output); + protobuf_c_message_free_unpacked(message, NULL); + + cfl_variant_destroy(resource); + cfl_variant_destroy(scope); + cfl_variant_destroy(original_item); + cfl_variant_destroy(result); + + ctx->instructions_left = ctx->instruction_limit; + lua_gc(l, LUA_GCCOLLECT, 0); + + if (errors > 5) { + flb_plg_warn(ins, "%d additional telemetry policy errors in this chunk", errors - 5); + } + + return errors; + +failure: + if (message) { + protobuf_c_message_free_unpacked(message, NULL); + } + + cfl_variant_destroy(resource); + cfl_variant_destroy(scope); + cfl_variant_destroy(original_item); + cfl_variant_destroy(result); + cfl_variant_destroy(groups); + cfl_variant_destroy(value); + cfl_variant_destroy(pending); + + ctx->instructions_left = ctx->instruction_limit; + lua_gc(l, LUA_GCCOLLECT, 0); + + flb_free(*output); + *output = NULL; + *output_size = 0; + + flb_plg_error(ins, "cannot construct FTS telemetry output (%s); edits discarded", phase); + return -1; +} diff --git a/plugins/processor_fts/fts_telemetry.h b/plugins/processor_fts/fts_telemetry.h new file mode 100644 index 00000000000..425b4131c62 --- /dev/null +++ b/plugins/processor_fts/fts_telemetry.h @@ -0,0 +1,40 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_FTS_TELEMETRY_H +#define FLB_FTS_TELEMETRY_H + +#include "fts_proto.h" + +extern const struct fts_signal_adapter fts_metrics_adapter; +extern const struct fts_signal_adapter fts_traces_adapter; + +/* shared metrics and traces policy path */ +int fts_register_telemetry_helpers(lua_State *l); +int fts_run_telemetry(lua_State *l); +int fts_telemetry_transform(struct flb_processor_instance *ins, char *buffer, size_t size, + char **output, size_t *output_size, const char *tag, int tag_len); + +/* signal entry points */ +int fts_process_metrics(struct flb_processor_instance *ins, struct cmt *input, + struct cmt **output, const char *tag, int tag_len); +int fts_process_traces(struct flb_processor_instance *ins, struct ctrace *input, + struct ctrace **output, const char *tag, int tag_len); + +#endif diff --git a/plugins/processor_fts/fts_telemetry_helpers.lua b/plugins/processor_fts/fts_telemetry_helpers.lua new file mode 100644 index 00000000000..0cad262021b --- /dev/null +++ b/plugins/processor_fts/fts_telemetry_helpers.lua @@ -0,0 +1,571 @@ +local fb = package.loaded.fluentbit +local kinds = {'gauge', 'sum', 'histogram', 'exponential_histogram', 'summary'} +local integer_properties = {kind=true, flags=true, dropped_attributes_count=true, + dropped_events_count=true, dropped_links_count=true} + +local function hex(bytes) + return (bytes:gsub('.', function(c) return string.format('%02x', c:byte()) end)) +end + +local function unhex(text, size, optional) + assert(type(text) == 'string', 'ID must be a hexadecimal string') + if optional and text == '' then return fb.bytes('') end + assert(#text == size * 2 and text:match('^[%x]+$') and not text:match('^0+$'), 'invalid ID') + return fb.bytes((text:gsub('..', function(c) return string.char(tonumber(c, 16)) end))) +end + +local function object(raw) + return setmetatable({}, { + __index=function(_, key) + local value = raw:get(key) + if not value then return nil end + local kind = value:type() + if key == 'trace_id' or key == 'span_id' or key == 'parent_span_id' then + return hex(value:value()) + end + if kind == 'string' or kind == 'bool' or kind == 'double' then return value:value() end + if integer_properties[key] then return value:number() end + return value + end, + __newindex=function(_, key, value) + if key == 'trace_id' or key == 'span_id' or key == 'parent_span_id' then + value = unhex(value, key == 'trace_id' and 16 or 8, key == 'parent_span_id') + elseif integer_properties[key] and type(value) == 'number' then + value = fb.int64(value) + end + raw:set(key, value) + end, + __metatable='FTS telemetry object' + }) +end + +local roots = setmetatable({}, {__mode='k'}) + +local function metric_data(item) + for _, kind in ipairs(kinds) do + local data = item:get(kind) + if data then return kind, data end + end + error('metric has no supported data type') +end + +function fb._fts_wrap(root, signal, emissions, tag) + local item = root:get('item') + local view = object(item) + local function context(key) + local fields = object(root:get(key)) + return setmetatable({}, { + __index=function(_, name) + if name == 'schema_url' then return root:get_string(key .. '_schema_url') end + return fields[name] + end, + __newindex=function(_, name, value) + assert(name ~= 'attributes', 'change context attributes through set/remove') + if name == 'schema_url' then root:set(key .. '_schema_url', value) + else fields[name] = value end + end, + __metatable='FTS context' + }) + end + local resource, scope = context('resource'), context('scope') + local proxy = setmetatable({}, { + __index=function(_, key) + if key == 'signal' then return signal end + if key == 'tag' then return tag end + if key == 'resource' then return resource end + if key == 'scope' then return scope end + if signal == 'metrics' then + local kind, data = metric_data(item) + if key == 'type' then return kind end + if key == 'datapoints' then return data:get('data_points') end + if key == 'aggregation_temporality' then + local value = data:get(key) + return value and ({[0]='unspecified',[1]='delta',[2]='cumulative'})[value:number()] + end + if key == 'is_monotonic' then + local value = data:get(key); return value and value:value() + end + if key == 'emit' then + return function() + assert(emissions, 'metric emission is unavailable') + emissions:append(root) + return fb._fts_wrap(emissions:get(emissions:len()), signal, nil, tag) + end + end + end + return view[key] + end, + __newindex=function(_, key, value) + assert(key ~= 'resource' and key ~= 'scope' and key ~= 'signal' and key ~= 'tag', + 'change context fields through their existing views') + if signal == 'metrics' then + local kind, data = metric_data(item) + if key == 'datapoints' then data:set('data_points', value); return end + if key == 'aggregation_temporality' then + local number = ({unspecified=0,delta=1,cumulative=2})[value] + assert(number and kind ~= 'gauge' and kind ~= 'summary', 'invalid temporality') + data:set(key, fb.int64(number)); return + end + if key == 'is_monotonic' then + assert(kind == 'sum' and type(value) == 'boolean', 'monotonicity requires a sum') + data:set(key, value); return + end + if key == 'type' then + assert((kind == 'gauge' or kind == 'sum') and (value == 'gauge' or value == 'sum'), + 'only gauge/sum conversion preserves the datapoint model') + if value == kind then return end + local points = data:get('data_points') + local converted = fb.map(); converted:set('data_points', points) + if value == 'sum' then + converted:set('aggregation_temporality', fb.int64(2)) + converted:set('is_monotonic', false) + end + item:remove(kind); item:set(value, converted); return + end + end + view[key] = value + end, + __metatable='FTS telemetry record' + }) + roots[proxy] = root + return proxy +end + +function fb.metric_value(point) + return point:get('as_int') or point:get('as_double') +end + +function fb.set_metric_value(point, value) + if type(value) == 'number' then + assert(value == value and value ~= math.huge and value ~= -math.huge, 'nonfinite metric edit') + point:remove('as_int'); point:set('as_double', value) + else + local kind = value:type() + assert(kind == 'int64' or kind == 'uint64' or kind == 'double', 'metric value must be numeric') + if kind == 'double' then point:remove('as_int'); point:set('as_double', value) + else point:remove('as_double'); point:set('as_int', value) end + end +end + +function fb.copy_metric(metric, name, description, unit) + local copy = metric:emit() + if name then copy.name = name end + if description then copy.description = description end + if unit then copy.unit = unit end + return copy +end + +function fb.scale_metric(metric, factor, unit) + assert(type(factor) == 'number' and factor > 0 and factor < math.huge, 'scale must be finite and positive') + assert(metric.type ~= 'exponential_histogram', 'exponential histogram scaling requires bucket remapping') + for i=1,metric.datapoints:len() do + local point = metric.datapoints:get(i) + local value = fb.metric_value(point) + if value then + local number = value:number() * factor + if value:type() == 'int64' and factor == math.floor(factor) then + fb.set_metric_value(point, fb.int64(number)) + else fb.set_metric_value(point, number) end + else + for _, key in ipairs({'sum','min','max'}) do + local number = point:get_number(key) + if number then point:set(key, number * factor) end + end + local bounds = point:get('explicit_bounds') + if bounds then + for j=1,bounds:len() do bounds:set(j, bounds:get(j):number() * factor) end + end + local quantiles = point:get('quantile_values') + if quantiles then + for j=1,quantiles:len() do + local quantile = quantiles:get(j) + quantile:set('value', quantile:get_number('value') * factor) + end + end + end + local exemplars = point:get('exemplars') + if exemplars then + for j=1,exemplars:len() do + local exemplar = exemplars:get(j) + local original = fb.metric_value(exemplar) + if original then + local scaled = original:number() * factor + if original:type() == 'int64' and factor == math.floor(factor) then + fb.set_metric_value(exemplar, fb.int64(scaled)) + else + fb.set_metric_value(exemplar, scaled) + end + end + end + end + end + if unit then metric.unit = unit end +end + +function fb.extract_metric(metric, field, name, monotonic) + assert(field == 'count' or field == 'sum' or field == 'avg', 'unsupported extraction') + assert(metric.type == 'histogram' or metric.type == 'exponential_histogram' or metric.type == 'summary', + 'extraction requires a distribution') + local points = fb.array() + for i=1,metric.datapoints:len() do + local source = metric.datapoints:get(i) + local value = source:get(field == 'avg' and 'sum' or field) + local count = source:get('count') + if value and (field ~= 'avg' or count:number() ~= 0) then + local point = fb.map() + point:set('attributes', source:get('attributes')) + for _, key in ipairs({'time_unix_nano','start_time_unix_nano','flags'}) do + local v = source:get(key); if v then point:set(key, v) end + end + if field == 'avg' then fb.set_metric_value(point, value:number() / count:number()) + elseif field == 'count' then fb.set_metric_value(point, value) + else fb.set_metric_value(point, value:number()) end + points:append(point) + end + end + if points:len() == 0 then return nil end + -- The derived family has a fresh number-datapoint model, preserving source context. + local copy = metric:emit() + copy.name = name or metric.name .. '.' .. field + -- Distribution conversion is internal to this helper, never an implicit reinterpretation. + fb._fts_number_metric(copy, points, field, metric.aggregation_temporality or 'cumulative') + if field == 'count' then copy.unit = '1' end + if monotonic ~= nil and field ~= 'avg' then copy.is_monotonic = monotonic end + return copy +end + +function fb._fts_number_metric(metric, points, field, temporality) + local item = roots[metric]:get('item') + for _, key in ipairs(kinds) do item:remove(key) end + local data = fb.map(); data:set('data_points', points) + if field == 'avg' then item:set('gauge', data) + else + data:set('aggregation_temporality', fb.int64(({unspecified=0,delta=1,cumulative=2})[temporality or 'cumulative'])) + data:set('is_monotonic', field == 'count') + item:set('sum', data) + end +end + +local function unsigned_time(value) + assert(value and (value:type() == 'uint64' or value:type() == 'int64'), 'time must be an integer') + local text = value:value() + assert(text:sub(1,1) ~= '-', 'time must be nonnegative') + return text +end + +local function after_or_equal(a, b) + return #a > #b or (#a == #b and a >= b) +end + +local function bucket_total(values) + local total = fb.uint64(0) + assert(values:type() == 'array', 'bucket counts must be an array') + for i=1,values:len() do + total = fb._fts_add(total, fb.uint64(unsigned_time(values:get(i)))) + end + return total +end + +local function valid_id(value, size, optional) + assert(value and value:type() == 'bytes', 'ID must be bytes') + local text = value:value() + assert((optional and #text == 0) or (#text == size and not text:match('^%z+$')), 'invalid ID') +end + +local function typed_identity(value) + local kind, parts = value:type(), {} + if kind == 'map' then + local keys = value:keys(); table.sort(keys) + for _, key in ipairs(keys) do + local child = typed_identity(value:get(key)) + parts[#parts+1] = #key .. ':' .. key .. #child .. ':' .. child + end + elseif kind == 'array' then + for i=1,value:len() do + local child = typed_identity(value:get(i)) + parts[#parts+1] = #child .. ':' .. child + end + elseif kind == 'null' then + return 'null' + else + local text = kind == 'double' and string.format('%.17g', value:number()) or tostring(value:value()) + if kind == 'int64' or kind == 'uint64' then kind = 'integer' end + return kind .. ':' .. #text .. ':' .. text + end + return kind .. ':' .. table.concat(parts) +end + +function fb._fts_validate_telemetry(root, signal, original) + local item = root:get('item') + assert(root:get('resource'):get('attributes'):type() == 'map', 'resource attributes must be a map') + assert(root:get('scope'):get('attributes'):type() == 'map', 'scope attributes must be a map') + local before = original and original:get("item") + local function changed(key) + local a, b = item:get(key), before and before:get(key) + return not before or (a == nil) ~= (b == nil) or (a and not fb._fts_equal(a, b)) + end + if signal == 'traces' then + if changed("kind") then + local kind = item:get_number("kind") + assert(kind >= 0 and kind <= 5 and kind % 1 == 0, "invalid span kind") + end + if changed("status") then + local status = item:get("status") + local code = status and status:get_number("code") or 0 + assert(code >= 0 and code <= 2 and code % 1 == 0, "invalid span status") + end + for _, key in ipairs({'trace_id','span_id','parent_span_id'}) do + if changed(key) then + local value = item:get(key) + assert(value and value:type() == 'bytes', 'span ID must be bytes internally') + local text = value:value() + assert((key == 'parent_span_id' and #text == 0) or + (#text == (key == 'trace_id' and 16 or 8) and not text:match('^%z+$')), 'invalid span ID') + end + end + if changed("end_time_unix_nano") or changed("start_time_unix_nano") then + assert(after_or_equal(unsigned_time(item:get('end_time_unix_nano')), + unsigned_time(item:get('start_time_unix_nano'))), 'span ends before it starts') + end + assert(item:get('attributes'):type() == 'map', 'span attributes must be a map') + local events, links = item:get('events'), item:get('links') + assert(events and events:type() == 'array' and links and links:type() == 'array', + 'span events and links must be arrays') + for i=1,events:len() do + local event = events:get(i) + assert(event:get('attributes'):type() == 'map', 'event attributes must be a map') + unsigned_time(event:get('time_unix_nano')) + end + for i=1,links:len() do + local link = links:get(i) + valid_id(link:get('trace_id'), 16, false); valid_id(link:get('span_id'), 8, false) + assert(link:get('attributes'):type() == 'map', 'link attributes must be a map') + end + else + local kind, data = metric_data(item) + if changed('name') then + assert(item:get_string('name') ~= '', 'metric name must not be empty') + end + local points = data:get('data_points') + assert(points and points:type() == 'array', 'datapoints must be an array') + local previous_kind, previous_data + if before then previous_kind, previous_data = metric_data(before) end + local previous_points = previous_data and previous_data:get('data_points') + local layout, quantile_layout, identities = nil, nil, {} + for i=1,points:len() do + local point = points:get(i) + local previous_point = previous_kind == kind and previous_points and previous_points:get(i) + local point_changes = {} + local function point_changed(key) + if point_changes[key] ~= nil then return point_changes[key] end + local a, b = point:get(key), previous_point and previous_point:get(key) + local result = not previous_point or (a == nil) ~= (b == nil) or + (a ~= nil and not fb._fts_equal(a, b)) + point_changes[key] = result + return result + end + local attrs = point:get('attributes') + assert(attrs and attrs:type() == 'map', 'datapoint attributes must be a map') + for _, key in ipairs(attrs:keys()) do + assert(key ~= '' or (previous_point and + previous_point:get('attributes'):get(key)), 'metric attribute keys must not be empty') + end + local finish = unsigned_time(point:get('time_unix_nano')) + local start = unsigned_time(point:get('start_time_unix_nano')) + if point_changed('time_unix_nano') or point_changed('start_time_unix_nano') then + assert(after_or_equal(finish, start), 'datapoint ends before it starts') + end + local keys = attrs:keys(); table.sort(keys) + local identity = start .. ":" .. finish + for _, key in ipairs(keys) do + local value = typed_identity(attrs:get(key)) + identity = identity .. '|' .. #key .. ':' .. key .. #value .. ':' .. value + end + if point_changed('attributes') or point_changed('time_unix_nano') then + assert(not identities[identity], 'duplicate series/window; aggregate explicitly') + end + identities[identity] = true + if (kind == 'gauge' or kind == 'sum') and + (point_changed('as_int') or point_changed('as_double') or + (kind == 'sum' and (not previous_data or not + fb._fts_equal(data:get('is_monotonic'), previous_data:get('is_monotonic'))))) then + local integer, number = point:get('as_int'), point:get('as_double') + assert((integer ~= nil) ~= (number ~= nil), 'scalar datapoint needs exactly one value') + if kind == 'sum' and data:get('is_monotonic'):value() then + local value = integer or number + if value:type() == 'int64' then + assert(not value:value():match('^-'), 'monotonic sum cannot be negative') + elseif value:type() == 'double' then + assert(value:number() >= 0, 'monotonic sum cannot be negative') + end + end + end + if kind == 'histogram' then + local bounds, counts = point:get('explicit_bounds'), point:get('bucket_counts') + assert(bounds and bounds:type() == 'array' and counts and counts:type() == 'array', + 'histogram bounds and counts must be arrays') + if point_changed('explicit_bounds') or point_changed('bucket_counts') or point_changed('count') then + assert(counts:len() == bounds:len()+1, 'histogram bucket layout mismatch') + assert(bucket_total(counts):value() == unsigned_time(point:get('count')), 'histogram count mismatch') + end + local previous, current = -math.huge, {} + for j=1,bounds:len() do + local boundary = bounds:get(j):number() + if point_changed('explicit_bounds') then + assert(boundary > previous and boundary < math.huge, 'histogram bounds must increase') + end + previous = boundary; current[j] = boundary + end + if layout then + assert(#layout == #current, 'native histogram layouts must match') + for j=1,#layout do assert(layout[j] == current[j], 'native histogram layouts must match') end + else layout = current end + elseif kind == 'exponential_histogram' then + local positive, negative = point:get('positive'), point:get('negative') + local positive_counts = positive and positive:get('bucket_counts') or fb.array() + local negative_counts = negative and negative:get('bucket_counts') or fb.array() + assert(positive_counts:len() <= 65535 and negative_counts:len() <= 65535, 'too many exponential buckets') + local total = fb._fts_add(bucket_total(positive_counts), bucket_total(negative_counts)) + total = fb._fts_add(total, fb.uint64(unsigned_time(point:get('zero_count')))) + if point_changed('positive') or point_changed('negative') or + point_changed('zero_count') or point_changed('count') then + assert(total:value() == unsigned_time(point:get('count')), 'exponential histogram count mismatch') + end + local scale = point:get('scale'):number() + if point_changed('scale') then + assert(scale >= -10 and scale <= 20, 'unsupported exponential scale') + end + if point_changed('zero_threshold') then + assert(point:get_number('zero_threshold') >= 0, 'negative zero threshold') + end + elseif kind == 'summary' then + local quantiles = point:get('quantile_values') + assert(quantiles and quantiles:type() == 'array', 'summary quantiles must be an array') + local previous, current = -1, {} + for j=1,quantiles:len() do + local quantile = quantiles:get(j):get_number('quantile') + if point_changed('quantile_values') then + assert(quantile >= 0 and quantile <= 1 and quantile > previous, 'invalid summary quantile') + end + previous = quantile; current[j] = quantile + end + if quantile_layout then + assert(#quantile_layout == #current, 'native summary layouts must match') + for j=1,#current do + assert(quantile_layout[j] == current[j], 'native summary layouts must match') + end + else quantile_layout = current end + end + end + end +end + +local function clone(value) + local holder = fb.map(); holder:set('value', value); return holder:get('value') +end + +local function identity(point, keys) + local attrs = point:get('attributes') + if keys then fb.keep_keys(attrs, keys) end + local names = attrs:keys(); table.sort(names) + local parts = {unsigned_time(point:get('start_time_unix_nano')), unsigned_time(point:get('time_unix_nano'))} + for _, key in ipairs(names) do + local value = typed_identity(attrs:get(key)) + parts[#parts+1] = #key .. ':' .. key .. #value .. ':' .. value + end + return table.concat(parts, '|') +end + +function fb.aggregate_metric(metric, keys, operation) + operation = operation or 'sum' + assert(metric.type == 'gauge' or metric.type == 'sum' or metric.type == 'histogram', + 'aggregation supports gauges, sums and explicit histograms') + assert(operation == 'sum' or (metric.type == 'gauge' and + (operation == 'min' or operation == 'max' or operation == 'mean' or operation == 'count')), + 'unsupported aggregation for this metric type') + local groups, ordered = {}, {} + for i=1,metric.datapoints:len() do + local point = clone(metric.datapoints:get(i)) + local key = identity(point, keys) + local state = groups[key] + if not state then + state = {point=point, count=1}; groups[key] = state; ordered[#ordered+1] = state + else + local target = state.point + assert((point:get('flags') and point:get('flags'):value()) == + (target:get('flags') and target:get('flags'):value()), 'aggregation flags differ') + state.count = state.count + 1 + if metric.type == 'histogram' then + local bounds, old = point:get('explicit_bounds'), target:get('explicit_bounds') + assert(bounds:len() == old:len(), 'histogram layouts differ') + for j=1,bounds:len() do + assert(bounds:get(j):value() == old:get(j):value(), 'histogram layouts differ') + end + local buckets, previous = point:get('bucket_counts'), target:get('bucket_counts') + for j=1,buckets:len() do previous:set(j, fb._fts_add(previous:get(j), buckets:get(j))) end + target:set('count', fb._fts_add(target:get('count'), point:get('count'))) + if target:get('sum') and point:get('sum') then + target:set('sum', fb._fts_add(target:get('sum'), point:get('sum'))) + else target:remove('sum') end + for _, bound in ipairs({'min','max'}) do + local a, b = target:get_number(bound), point:get_number(bound) + if a and b then target:set(bound, (bound == 'min' and math.min or math.max)(a,b)) + else target:remove(bound) end + end + else + local a, b = fb.metric_value(target), fb.metric_value(point) + if operation == 'sum' or operation == 'mean' then + fb.set_metric_value(target, fb._fts_add(a,b)) + elseif operation == 'min' or operation == 'max' then + if (operation == 'min' and b:number() < a:number()) or + (operation == 'max' and b:number() > a:number()) then fb.set_metric_value(target,b) end + end + end + local exemplars = point:get('exemplars') + if exemplars then + local target_exemplars = target:get('exemplars') or fb.array() + for j=1,exemplars:len() do target_exemplars:append(exemplars:get(j)) end + target:set('exemplars', target_exemplars) + end + end + end + local output = fb.array() + for _, state in ipairs(ordered) do + if operation == 'mean' then fb.set_metric_value(state.point, fb.metric_value(state.point):number() / state.count) + elseif operation == 'count' then fb.set_metric_value(state.point, fb.int64(state.count)) end + output:append(state.point) + end + metric.datapoints = output + if operation == 'count' then metric.unit = '1' end +end + +function fb.merge_histogram_buckets(metric, bounds) + assert(metric.type == 'histogram', 'bucket merging requires an explicit histogram') + if type(bounds) == 'userdata' then + assert(bounds:type() == 'array', 'bounds must be an array') + local values = {} + for i=1,bounds:len() do values[i] = bounds:get(i):number() end + bounds = values + end + assert(type(bounds) == 'table', 'bounds must be a numeric array') + for i=2,#bounds do assert(bounds[i] > bounds[i-1], 'bounds must increase') end + for i=1,metric.datapoints:len() do + local point = metric.datapoints:get(i) + local old, counts = point:get('explicit_bounds'), point:get('bucket_counts') + local output, new_bounds = fb.array(), fb.array() + local at, total = 1, fb.uint64(0) + for _, boundary in ipairs(bounds) do + local found = false + while at <= old:len() and old:get(at):number() <= boundary do + total = fb._fts_add(total, counts:get(at)) + if old:get(at):number() == boundary then found = true end + at = at + 1 + end + assert(found, 'new bounds must be a subset of existing bounds') + new_bounds:append(boundary); output:append(total); total = fb.uint64(0) + end + while at <= counts:len() do total = fb._fts_add(total, counts:get(at)); at = at + 1 end + output:append(total) + point:set('explicit_bounds', new_bounds); point:set('bucket_counts', output) + end +end diff --git a/plugins/processor_fts/fts_traces.c b/plugins/processor_fts/fts_traces.c new file mode 100644 index 00000000000..8e660c34a51 --- /dev/null +++ b/plugins/processor_fts/fts_traces.c @@ -0,0 +1,143 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include "fts_telemetry.h" + +const struct fts_signal_adapter fts_traces_adapter = { + .event_type = FLB_PROCESSOR_TRACES, + .register_helpers = fts_register_telemetry_helpers, + .run_callback = fts_run_telemetry +}; + +/* + * Trace processors commit in place: the framework retains the original + * context pointer, so staged resource spans are moved into it. + */ +static void commit_trace(struct ctrace *input, struct ctrace *staged) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct ctrace_resource_span *resource; + struct ctrace_span *span; + + cfl_list_foreach_safe(head, tmp, &input->resource_spans) { + resource = cfl_list_entry(head, struct ctrace_resource_span, _head); + ctr_resource_span_destroy(resource); + } + + cfl_list_foreach_safe(head, tmp, &staged->resource_spans) { + resource = cfl_list_entry(head, struct ctrace_resource_span, _head); + cfl_list_del(head); + cfl_list_add(head, &input->resource_spans); + resource->ctx = input; + } + + cfl_list_foreach_safe(head, tmp, &staged->span_list) { + span = cfl_list_entry(head, struct ctrace_span, _head_global); + cfl_list_del(head); + cfl_list_add(head, &input->span_list); + span->ctx = input; + } + + input->last_span_id = staged->last_span_id; + ctr_destroy(staged); +} + +int fts_process_traces(struct flb_processor_instance *ins, struct ctrace *input, + struct ctrace **output, const char *tag, int tag_len) +{ + struct fts_runtime *ctx = ins->context; + cfl_sds_t encoded; + char *buffer = NULL; + size_t size; + size_t offset = 0; + int ret; + int handled_errors = 0; + const char *phase = "encode native traces"; + + *output = NULL; + + encoded = ctr_encode_opentelemetry_create(input); + if (!encoded) { + goto failure; + } + + phase = "transform span policies"; + ret = fts_telemetry_transform(ins, encoded, cfl_sds_len(encoded), &buffer, &size, + tag, tag_len); + ctr_encode_opentelemetry_destroy(encoded); + handled_errors = ret > 0; + if (ret < 0) { + goto failure; + } + + if (size == 0) { + flb_free(buffer); + *output = ctr_create(NULL); + if (!*output) { + return FLB_PROCESSOR_FAILURE; + } + commit_trace(input, *output); + *output = input; + + if (handled_errors) { + fts_report_policy_error(ins); + } + return FLB_PROCESSOR_SUCCESS; + } + + phase = "decode transformed traces"; + ret = ctr_decode_opentelemetry_create(output, buffer, size, &offset); + flb_free(buffer); + buffer = NULL; + if (ret != 0 || offset != size || !*output) { + ctr_destroy(*output); + *output = NULL; + goto failure; + } + + commit_trace(input, *output); + *output = input; + + if (handled_errors) { + fts_report_policy_error(ins); + } + + return FLB_PROCESSOR_SUCCESS; + +failure: + fts_report_policy_error(ins); + flb_free(buffer); + flb_plg_error(ins, "cannot construct FTS traces output (%s); edits discarded", phase); + + if (strcmp(ctx->error_mode, "drop") == 0) { + *output = ctr_create(NULL); + if (!*output) { + return FLB_PROCESSOR_FAILURE; + } + commit_trace(input, *output); + } + + *output = input; + return FLB_PROCESSOR_SUCCESS; +} diff --git a/plugins/processor_fts/fts_value_helpers.lua b/plugins/processor_fts/fts_value_helpers.lua new file mode 100644 index 00000000000..50c6bf10288 --- /dev/null +++ b/plugins/processor_fts/fts_value_helpers.lua @@ -0,0 +1,37 @@ +local fb = package.loaded.fluentbit +function fb.keep_keys(map, keys) + local keep = {}; for _, k in ipairs(keys) do keep[k] = true end + for _, k in ipairs(map:keys()) do if not keep[k] then map:remove(k) end end +end +function fb.parse_time(text) + assert(type(text) == 'string', 'timestamp must be RFC3339 text') + local y,m,d,h,mi,s,tail = text:match('^(%d%d%d%d)%-(%d%d)%-(%d%d)[Tt](%d%d):(%d%d):(%d%d)(.*)$') + assert(y, 'invalid RFC3339 timestamp') + y,m,d,h,mi,s = tonumber(y),tonumber(m),tonumber(d),tonumber(h),tonumber(mi),tonumber(s) + local leap = y%4 == 0 and (y%100 ~= 0 or y%400 == 0) + local lengths = {31,leap and 29 or 28,31,30,31,30,31,31,30,31,30,31} + assert(y>=1969 and y<=2107 and m>=1 and m<=12 and d>=1 and d<=lengths[m] + and h<24 and mi<60 and s<60, 'invalid calendar timestamp') + local fraction = tail:match('^%.(%d+)') + local nsec = 0 + if fraction then + assert(#fraction <= 9, 'timestamp exceeds nanosecond precision') + nsec = tonumber(fraction .. string.rep('0', 9 - #fraction)) + tail = tail:sub(#fraction + 2) + end + local offset = 0 + if tail ~= 'Z' and tail ~= 'z' then + local sign,oh,om = tail:match('^([+-])(%d%d):(%d%d)$') + assert(sign and tonumber(oh)<24 and tonumber(om)<60, 'invalid timestamp timezone') + offset = (tonumber(oh)*60 + tonumber(om))*60*(sign == '+' and 1 or -1) + end + local days = 0 + if y == 1969 then days = -365 end + for year=1970,y-1 do + days = days + ((year%4 == 0 and (year%100 ~= 0 or year%400 == 0)) and 366 or 365) + end + for month=1,m-1 do days = days + lengths[month] end + local sec = (days+d-1)*86400+h*3600+mi*60+s-offset + assert(sec>=0 and sec<=4294967293, 'timestamp outside log event range') + return {sec=sec,nsec=nsec} +end diff --git a/plugins/processor_fts/fts_values.c b/plugins/processor_fts/fts_values.c new file mode 100644 index 00000000000..77701883b1a --- /dev/null +++ b/plugins/processor_fts/fts_values.c @@ -0,0 +1,2202 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fts.h" +#include "fts_helpers.h" + +#define VALUE_MT "fluentbit.typed_value" +#define SAFE_INTEGER 9007199254740991.0 + +/* Empty input maps need no native allocation until a policy writes to them. + * This immutable backing is never attached to a mutable CFL container. */ +static struct cfl_kvlist empty_map = { + .list = {&empty_map.list, &empty_map.list} +}; +static struct cfl_variant empty_value = { + .type = CFL_VARIANT_KVLIST, + .data.as_kvlist = &empty_map +}; + +static struct fts_value *push_value(lua_State *l, struct cfl_variant *v, int readonly, + int borrowed); + +/* Record handles may borrow chunk storage, so they are valid only during the + * callback that created them. copy() returns a value that can be kept. */ +static struct fts_value *check_live(lua_State *l, struct fts_value *value) +{ + if (value->record && value->generation != value->ctx->generation) { + luaL_error(l, "log value used after its callback; use copy() to keep it"); + } + + return value; +} + +/* Only push_value() creates userdata of this exact size, and it stamps the + * private runtime pointer; policies cannot forge either. Comparing them avoids + * a metatable round trip on every typed call. */ +static struct fts_value *check_value(lua_State *l, int index) +{ + struct fts_runtime *ctx; + struct fts_value *value; + + lua_getallocf(l, (void **) &ctx); + + value = NULL; + if (lua_type(l, index) == LUA_TUSERDATA && lua_objlen(l, index) == sizeof(*value)) { + value = lua_touserdata(l, index); + } + + if (!value || value->ctx != ctx) { + luaL_argerror(l, index, "typed value expected"); + return NULL; + } + + check_live(l, value); + + return value; +} + +/* Only adapter callbacks call these: their closures carry the key strings + * as upvalues, in fts_key order. */ +void fts_getfield(lua_State *l, int index, enum fts_key key) +{ + if (index < 0 && index > LUA_REGISTRYINDEX) { + index = lua_gettop(l) + index + 1; + } + + lua_pushvalue(l, lua_upvalueindex(key + 1)); + lua_rawget(l, index); +} + +void fts_setfield(lua_State *l, int index, enum fts_key key) +{ + if (index < 0 && index > LUA_REGISTRYINDEX) { + index = lua_gettop(l) + index + 1; + } + + lua_pushvalue(l, lua_upvalueindex(key + 1)); + lua_insert(l, -2); + lua_rawset(l, index); +} + +/* Compare stored entries, preserving duplicate keys and their order. */ +int fts_values_equal(struct cfl_variant *a, struct cfl_variant *b, int depth) +{ + struct cfl_list *left; + struct cfl_list *right; + struct cfl_kvpair *x; + struct cfl_kvpair *y; + size_t i; + + if (a == b) { + return FLB_TRUE; + } + + if (!a || !b || a->type != b->type || depth >= FTS_MAX_DEPTH) { + return FLB_FALSE; + } + + switch (a->type) { + case CFL_VARIANT_NULL: + return FLB_TRUE; + case CFL_VARIANT_BOOL: + return a->data.as_bool == b->data.as_bool; + case CFL_VARIANT_INT: + return a->data.as_int64 == b->data.as_int64; + case CFL_VARIANT_UINT: + return a->data.as_uint64 == b->data.as_uint64; + case CFL_VARIANT_DOUBLE: + return memcmp(&a->data.as_double, &b->data.as_double, sizeof(double)) == 0; + case CFL_VARIANT_STRING: + case CFL_VARIANT_BYTES: + return cfl_variant_size_get(a) == cfl_variant_size_get(b) && + memcmp(a->data.as_string, b->data.as_string, cfl_variant_size_get(a)) == 0; + case CFL_VARIANT_ARRAY: + if (a->data.as_array->entry_count != b->data.as_array->entry_count) { + return FLB_FALSE; + } + + for (i = 0; i < a->data.as_array->entry_count; i++) { + if (!fts_values_equal(a->data.as_array->entries[i], + b->data.as_array->entries[i], depth + 1)) { + return FLB_FALSE; + } + } + + return FLB_TRUE; + case CFL_VARIANT_KVLIST: + left = a->data.as_kvlist->list.next; + right = b->data.as_kvlist->list.next; + + while (left != &a->data.as_kvlist->list && right != &b->data.as_kvlist->list) { + x = cfl_list_entry(left, struct cfl_kvpair, _head); + y = cfl_list_entry(right, struct cfl_kvpair, _head); + if (cfl_sds_len(x->key) != cfl_sds_len(y->key) || + memcmp(x->key, y->key, cfl_sds_len(x->key)) != 0 || + !fts_values_equal(x->val, y->val, depth + 1)) { + return FLB_FALSE; + } + + left = left->next; + right = right->next; + } + + return left == &a->data.as_kvlist->list && right == &b->data.as_kvlist->list; + default: + return FLB_FALSE; + } +} + +static int equal_values(lua_State *l) +{ + lua_pushboolean(l, fts_values_equal(fts_resolve_value(l, 1, 0), + fts_resolve_value(l, 2, 0), 0)); + return 1; +} + +/* Arena copies are private native results. Script-visible snapshots use heap + * ownership so retained Lua values remain independent of chunk lifetimes. */ +struct cfl_variant *fts_copy_value_in(struct cfl_variant *v, int depth, struct cfl_arena *arena) +{ + struct cfl_variant *copy; + struct cfl_variant *child; + struct cfl_array *array; + struct cfl_kvlist *map; + struct cfl_list *head; + struct cfl_kvpair *pair; + size_t i; + size_t length; + + if (!v || depth >= FTS_MAX_DEPTH) { + return NULL; + } + + switch (v->type) { + case CFL_VARIANT_NULL: + return cfl_variant_create_from_null_in(arena); + case CFL_VARIANT_BOOL: + return cfl_variant_create_from_bool_in(arena, v->data.as_bool); + case CFL_VARIANT_INT: + return cfl_variant_create_from_int64_in(arena, v->data.as_int64); + case CFL_VARIANT_UINT: + return cfl_variant_create_from_uint64_in(arena, v->data.as_uint64); + case CFL_VARIANT_DOUBLE: + return cfl_variant_create_from_double_in(arena, v->data.as_double); + case CFL_VARIANT_STRING: + case CFL_VARIANT_BYTES: + if (arena) { + if (v->type == CFL_VARIANT_STRING) { + return cfl_variant_create_from_string_s_in(arena, v->data.as_string, + cfl_variant_size_get(v), CFL_FALSE); + } + return cfl_variant_create_from_bytes_in(arena, v->data.as_bytes, + cfl_variant_size_get(v), CFL_FALSE); + } + + /* Consumers outside FTS read string and bytes lengths through cfl_sds_len(), + * so heap copies always own a real sds payload. */ + length = cfl_variant_size_get(v); + if (length > INT_MAX) { + return NULL; + } + + if (v->type == CFL_VARIANT_STRING) { + copy = cfl_variant_create_from_string_s(v->data.as_string, length, CFL_FALSE); + } + else { + copy = cfl_variant_create_from_bytes(v->data.as_bytes, length, CFL_FALSE); + } + + if (!copy) { + flb_errno(); + } + return copy; + case CFL_VARIANT_ARRAY: + array = cfl_array_create_in(arena, v->data.as_array->entry_count); + if (!array) { + return NULL; + } + + for (i = 0; i < v->data.as_array->entry_count; i++) { + child = fts_copy_value_in(v->data.as_array->entries[i], depth + 1, arena); + if (!child || cfl_array_append(array, child) != 0) { + cfl_variant_destroy(child); + cfl_array_destroy(array); + return NULL; + } + } + + copy = cfl_variant_create_from_array_in(arena, array); + if (!copy) { + cfl_array_destroy(array); + } + return copy; + case CFL_VARIANT_KVLIST: + map = cfl_kvlist_create_in(arena); + if (!map) { + return NULL; + } + + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + child = fts_copy_value_in(pair->val, depth + 1, arena); + if (!child || + cfl_kvlist_insert_s(map, pair->key, cfl_sds_len(pair->key), child) != 0) { + cfl_variant_destroy(child); + cfl_kvlist_destroy(map); + return NULL; + } + } + + copy = cfl_variant_create_from_kvlist_in(arena, map); + if (!copy) { + cfl_kvlist_destroy(map); + } + return copy; + default: + return NULL; + } +} + +struct cfl_variant *fts_copy_value(struct cfl_variant *value, int depth) +{ + return fts_copy_value_in(value, depth, NULL); +} + +/* Conservative accounting includes container overhead and every owned scalar. */ +size_t fts_value_cost(struct cfl_variant *v, int depth) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + size_t total = 256; + size_t child; + size_t i; + + if (!v || depth >= FTS_MAX_DEPTH) { + return SIZE_MAX; + } + + if (v->type == CFL_VARIANT_STRING || v->type == CFL_VARIANT_BYTES) { + child = cfl_variant_size_get(v); + return child > SIZE_MAX - total ? SIZE_MAX : total + child; + } + + if (v->type == CFL_VARIANT_ARRAY) { + for (i = 0; i < v->data.as_array->entry_count; i++) { + child = fts_value_cost(v->data.as_array->entries[i], depth + 1); + if (child > SIZE_MAX - total) { + return SIZE_MAX; + } + total += child; + } + } + else if (v->type == CFL_VARIANT_KVLIST) { + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + child = cfl_sds_len(pair->key) + 128; + if (child > SIZE_MAX - total) { + return SIZE_MAX; + } + total += child; + + child = fts_value_cost(pair->val, depth + 1); + if (child > SIZE_MAX - total) { + return SIZE_MAX; + } + total += child; + } + } + + return total; +} + +void fts_charge_value(lua_State *l, struct fts_value *owner, size_t bytes) +{ + struct fts_runtime *ctx = owner->ctx; + + if (bytes > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - bytes || + ctx->lua_bytes > ctx->memory_limit - ctx->native_bytes - bytes) { + /* Native values are invisible to Lua's allocation-based GC pacing. */ + lua_gc(l, LUA_GCCOLLECT, 0); + } + + if (bytes > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - bytes || + ctx->lua_bytes > ctx->memory_limit - ctx->native_bytes - bytes) { + luaL_error(l, "Lua policy memory limit exceeded"); + } + + /* In-place additions become chunk data when the record callback ends. */ + if (owner->inplace) { + ctx->undo_charged += bytes; + } + else { + owner->charged += bytes; + } + + ctx->native_bytes += bytes; + if (bytes > SIZE_MAX - ctx->native_allocated) { + ctx->native_allocated = SIZE_MAX; + } + else { + ctx->native_allocated = ctx->native_allocated + bytes; + } +} + +/* No native ownership or reservation means no finalizer is needed. + * A first write installs normal ownership before allocating storage. */ +struct fts_value *fts_push_empty_map(lua_State *l) +{ + return fts_push_borrowed(l, &empty_value); +} + +/* Record roots read the chunk's tree directly; the first write copies it. */ +struct fts_value *fts_push_borrowed(lua_State *l, struct cfl_variant *source) +{ + struct fts_value *value; + + value = push_value(l, source, FLB_FALSE, FLB_TRUE); + value->record = FLB_TRUE; + value->generation = value->ctx->generation; + + return value; +} + +/* Values written into a container come from the same allocator as it. */ +static struct cfl_arena *container_arena(struct cfl_variant *container) +{ + if (container->type == CFL_VARIANT_KVLIST) { + return container->data.as_kvlist->arena; + } + + if (container->type == CFL_VARIANT_ARRAY) { + return container->data.as_array->arena; + } + + return NULL; +} + +/* Exact byte matching: OTel attribute names are case sensitive. */ +struct cfl_kvpair *fts_find_pair(struct cfl_variant *v, const char *key, size_t length) +{ + struct cfl_list *head; + struct cfl_kvpair *pair; + + if (v->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (cfl_sds_len(pair->key) == length && memcmp(pair->key, key, length) == 0) { + return pair; + } + } + + return NULL; +} + +/* Views retain their parent through their Lua environment. Resolve each use + * rather than storing child pointers that set/remove could invalidate. */ +struct cfl_variant *fts_resolve_value(lua_State *l, int index, int depth) +{ + struct fts_value *value; + struct cfl_variant *parent; + struct cfl_variant *result = NULL; + struct cfl_kvpair *pair; + const char *key; + size_t length; + lua_Number position; + + value = check_value(l, index); + if (value->owned) { + return value->owned; + } + + if (value->cache && value->epoch == value->root->epoch) { + check_live(l, value->root); + return value->cache; + } + + if (depth >= FTS_MAX_DEPTH || !lua_checkstack(l, 4)) { + luaL_error(l, "typed value nesting exceeds limit"); + } + + lua_getfenv(l, index); + lua_rawgeti(l, -1, 1); + parent = fts_resolve_value(l, -1, depth + 1); + lua_pop(l, 1); + + lua_rawgeti(l, -1, 2); + if (parent->type == CFL_VARIANT_KVLIST && lua_type(l, -1) == LUA_TSTRING) { + key = lua_tolstring(l, -1, &length); + pair = fts_find_pair(parent, key, length); + if (pair) { + result = pair->val; + } + } + else if (parent->type == CFL_VARIANT_ARRAY && lua_type(l, -1) == LUA_TNUMBER) { + position = lua_tonumber(l, -1); + if (position >= 1 && position <= parent->data.as_array->entry_count) { + result = parent->data.as_array->entries[(size_t) position - 1]; + } + } + lua_pop(l, 2); + + if (!result) { + luaL_error(l, "typed value path no longer exists"); + } + + value->cache = result; + value->epoch = value->root->epoch; + + return result; +} + +static struct fts_value *push_value(lua_State *l, struct cfl_variant *v, int readonly, + int borrowed) +{ + struct fts_value *value; + + value = lua_newuserdata(l, sizeof(*value)); + value->owned = v; + value->readonly = readonly; + value->dirty = FLB_FALSE; + value->depth_changed = !borrowed; + value->borrowed = borrowed; + value->record = FLB_FALSE; + value->inplace = FLB_FALSE; + value->generation = 0; + value->charged = 0; + value->root = NULL; + value->cache = NULL; + value->epoch = 0; + lua_getallocf(l, (void **) &value->ctx); + + /* Borrowed storage is not owned, so it needs no finalizer. */ + if (borrowed) { + lua_rawgeti(l, LUA_REGISTRYINDEX, value->ctx->empty_metatable_ref); + } + else { + lua_rawgeti(l, LUA_REGISTRYINDEX, value->ctx->value_metatable_ref); + } + lua_setmetatable(l, -2); + + return value; +} + +struct fts_value *fts_push_owned(lua_State *l, struct cfl_variant *v, int readonly) +{ + return push_value(l, v, readonly, FLB_FALSE); +} + +static void push_view(lua_State *l, int parent, int key) +{ + struct fts_value *owner; + struct fts_value *view; + + owner = check_value(l, parent); + if (!owner->owned && !owner->root) { + luaL_error(l, "typed value has no storage"); + } + + /* Views own no storage, so they use the finalizer-free metatable. */ + view = push_value(l, NULL, owner->readonly, FLB_TRUE); + view->borrowed = FLB_FALSE; + view->depth_changed = FLB_TRUE; + view->root = owner->owned ? owner : owner->root; + + /* The environment pins the parent; slots 1 and 2 hold parent and key. */ + lua_createtable(l, 2, 0); + lua_pushvalue(l, parent); + lua_rawseti(l, -2, 1); + lua_pushvalue(l, key); + lua_rawseti(l, -2, 2); + lua_setfenv(l, -2); +} + +static int value_gc(lua_State *l) +{ + struct fts_value *value; + + /* Only the VM can call this private finalizer. */ + value = lua_touserdata(l, 1); + if (!value->borrowed) { + cfl_variant_destroy(value->owned); + } + + value->owned = NULL; + value->ctx->native_bytes -= value->charged; + value->charged = 0; + + return 0; +} + +static int value_type(lua_State *l) +{ + struct cfl_variant *v; + const char *name; + + v = fts_resolve_value(l, 1, 0); + switch (v->type) { + case CFL_VARIANT_NULL: + name = "null"; + break; + case CFL_VARIANT_BOOL: + name = "bool"; + break; + case CFL_VARIANT_INT: + name = "int64"; + break; + case CFL_VARIANT_UINT: + name = "uint64"; + break; + case CFL_VARIANT_DOUBLE: + name = "double"; + break; + case CFL_VARIANT_STRING: + name = "string"; + break; + case CFL_VARIANT_BYTES: + name = "bytes"; + break; + case CFL_VARIANT_ARRAY: + name = "array"; + break; + case CFL_VARIANT_KVLIST: + name = "map"; + break; + default: + return luaL_error(l, "unsupported telemetry type"); + } + + lua_pushstring(l, name); + return 1; +} + +static int push_scalar(lua_State *l, struct cfl_variant *v) +{ + char buffer[32]; + + switch (v->type) { + case CFL_VARIANT_NULL: + lua_pushnil(l); + break; + case CFL_VARIANT_BOOL: + lua_pushboolean(l, v->data.as_bool); + break; + case CFL_VARIANT_DOUBLE: + lua_pushnumber(l, v->data.as_double); + break; + case CFL_VARIANT_STRING: + lua_pushlstring(l, v->data.as_string, cfl_variant_size_get(v)); + break; + case CFL_VARIANT_BYTES: + lua_pushlstring(l, v->data.as_bytes, cfl_variant_size_get(v)); + break; + case CFL_VARIANT_INT: + snprintf(buffer, sizeof(buffer), "%" PRId64, v->data.as_int64); + lua_pushstring(l, buffer); + break; + case CFL_VARIANT_UINT: + snprintf(buffer, sizeof(buffer), "%" PRIu64, v->data.as_uint64); + lua_pushstring(l, buffer); + break; + default: + return luaL_error(l, "value() requires a scalar; use get() for containers"); + } + + return 1; +} + +static int push_number(lua_State *l, struct cfl_variant *v) +{ + lua_Number number; + + if (v->type == CFL_VARIANT_INT) { + if (v->data.as_int64 < -INT64_C(9007199254740991) || + v->data.as_int64 > INT64_C(9007199254740991)) { + return luaL_error(l, "integer cannot be represented exactly as a Lua number"); + } + number = v->data.as_int64; + } + else if (v->type == CFL_VARIANT_UINT) { + if (v->data.as_uint64 > UINT64_C(9007199254740991)) { + return luaL_error(l, "integer cannot be represented exactly as a Lua number"); + } + number = v->data.as_uint64; + } + else if (v->type == CFL_VARIANT_DOUBLE) { + number = v->data.as_double; + } + else { + return luaL_error(l, "number() requires a numeric value"); + } + + lua_pushnumber(l, number); + return 1; +} + +static int value_scalar(lua_State *l) +{ + return push_scalar(l, fts_resolve_value(l, 1, 0)); +} + +static int value_number(lua_State *l) +{ + return push_number(l, fts_resolve_value(l, 1, 0)); +} + +static size_t array_index(lua_State *l, struct cfl_variant *v, int index) +{ + lua_Number position; + + if (lua_type(l, index) != LUA_TNUMBER) { + luaL_error(l, "array index must be a number"); + } + + position = lua_tonumber(l, index); + if (!isfinite(position) || position != floor(position) || position < 1 || + position > v->data.as_array->entry_count) { + luaL_error(l, "array index out of bounds (indices start at 1)"); + } + + return (size_t) position - 1; +} + +/* Resolve once without creating a Lua view for scalar reads. */ +static struct cfl_variant *get_child(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_kvpair *pair; + const char *key; + size_t length; + + v = fts_resolve_value(l, 1, 0); + if (v->type == CFL_VARIANT_KVLIST) { + if (lua_type(l, 2) != LUA_TSTRING) { + luaL_error(l, "map key must be a string"); + } + + key = lua_tolstring(l, 2, &length); + pair = fts_find_pair(v, key, length); + return pair ? pair->val : NULL; + } + + if (v->type == CFL_VARIANT_ARRAY) { + return v->data.as_array->entries[array_index(l, v, 2)]; + } + + luaL_error(l, "get() requires a map or array"); + return NULL; +} + +static int value_get(lua_State *l) +{ + if (!get_child(l)) { + lua_pushnil(l); + return 1; + } + + push_view(l, 1, 2); + return 1; +} + +/* Scalar convenience accessors retain strict types and missing-key semantics. */ +static int get_scalar(lua_State *l, int type) +{ + struct cfl_variant *v; + + v = get_child(l); + if (!v) { + lua_pushnil(l); + return 1; + } + + if (type == CFL_VARIANT_DOUBLE) { + return push_number(l, v); + } + + if (v->type != type) { + return luaL_error(l, "field has a different type than the requested scalar accessor"); + } + + return push_scalar(l, v); +} + +static int value_get_string(lua_State *l) +{ + return get_scalar(l, CFL_VARIANT_STRING); +} + +static int value_get_bool(lua_State *l) +{ + return get_scalar(l, CFL_VARIANT_BOOL); +} + +static int value_get_number(lua_State *l) +{ + return get_scalar(l, CFL_VARIANT_DOUBLE); +} + +enum { + UNDO_SET_PAIR, + UNDO_INSERT_PAIR, + UNDO_REMOVE_PAIR, + UNDO_SET_INDEX, + UNDO_APPEND +}; + +/* Reserve before mutating so a recorded edit can never be lost. */ +static void undo_reserve(lua_State *l, struct fts_runtime *ctx) +{ + struct fts_undo_entry *entries; + size_t size; + + if (ctx->undo_count < ctx->undo_size) { + return; + } + + size = ctx->undo_size ? ctx->undo_size * 2 : 64; + if (size > SIZE_MAX / sizeof(*entries)) { + luaL_error(l, "cannot allocate edit log"); + } + + entries = flb_realloc(ctx->undo, size * sizeof(*entries)); + if (!entries) { + flb_errno(); + luaL_error(l, "cannot allocate edit log"); + } + + ctx->undo = entries; + ctx->undo_size = size; +} + +static void undo_add(struct fts_runtime *ctx, int type, struct cfl_list *list, + struct cfl_kvpair *pair, struct cfl_list *prev, + struct cfl_array *array, size_t index, struct cfl_variant *old) +{ + struct fts_undo_entry *entry; + struct cfl_variant *held_value = old; + size_t held = 64; + size_t cost; + + entry = &ctx->undo[ctx->undo_count++]; + entry->type = type; + entry->list = list; + entry->pair = pair; + entry->prev = prev; + entry->array = array; + entry->index = index; + entry->old = old; + + if (type == UNDO_REMOVE_PAIR) { + held += 128 + cfl_sds_len(pair->key); + held_value = pair->val; + } + + if (held_value) { + cost = fts_value_cost(held_value, 0); + held = cost > SIZE_MAX - held ? SIZE_MAX : held + cost; + } + + ctx->undo_held = held > SIZE_MAX - ctx->undo_held ? SIZE_MAX : ctx->undo_held + held; +} + +/* Restore edits newer than mark, newest first. */ +void fts_undo_rollback(struct fts_runtime *ctx, size_t mark) +{ + struct fts_undo_entry *entry; + struct cfl_variant *current; + + while (ctx->undo_count > mark) { + entry = &ctx->undo[--ctx->undo_count]; + switch (entry->type) { + case UNDO_SET_PAIR: + current = entry->pair->val; + entry->pair->val = entry->old; + cfl_variant_destroy(current); + break; + case UNDO_INSERT_PAIR: + cfl_kvpair_destroy(entry->pair); + break; + case UNDO_REMOVE_PAIR: + /* cfl_list_add_after() leaves next->prev stale; insert before next. */ + cfl_list_add_before(&entry->pair->_head, entry->prev->next, entry->list); + break; + case UNDO_SET_INDEX: + current = entry->array->entries[entry->index]; + entry->array->entries[entry->index] = entry->old; + cfl_variant_destroy(current); + break; + case UNDO_APPEND: + current = entry->array->entries[--entry->array->entry_count]; + entry->array->entries[entry->array->entry_count] = NULL; + cfl_variant_destroy(current); + break; + } + } + + if (mark == 0) { + ctx->undo_held = 0; + } +} + +/* Release replaced and removed values once the chunk output is final. */ +void fts_undo_commit(struct fts_runtime *ctx) +{ + struct fts_undo_entry *entry; + size_t index; + + for (index = 0; index < ctx->undo_count; index++) { + entry = &ctx->undo[index]; + if (entry->type == UNDO_SET_PAIR || entry->type == UNDO_SET_INDEX) { + cfl_variant_destroy(entry->old); + } + else if (entry->type == UNDO_REMOVE_PAIR) { + cfl_kvpair_destroy(entry->pair); + } + } + + ctx->undo_count = 0; + ctx->undo_held = 0; +} + +static struct fts_value *root_value(lua_State *l, int index) +{ + struct fts_value *value; + + /* Public mutation entry points already verified the receiver. View + * environments are private and retain only verified parent values. */ + value = lua_touserdata(l, index); + return value->owned ? value : value->root; +} + +static size_t cost_add(size_t a, size_t b) +{ + return b > SIZE_MAX - a ? SIZE_MAX : a + b; +} + +static void credit_value(lua_State *l, struct fts_value *owner, size_t bytes) +{ + /* Replaced in-place data stays allocated in the undo log until commit. */ + if (owner->inplace) { + return; + } + + /* An unknown cost never releases reservations held by other values. */ + if (bytes == SIZE_MAX) { + return; + } + + if (bytes > owner->charged) { + bytes = owner->charged; + } + + owner->charged -= bytes; + owner->ctx->native_bytes -= bytes; +} + +static void charge_mutation(lua_State *l, int index, size_t overhead) +{ + size_t bytes = 256; + size_t length; + + if (lua_type(l, index) == LUA_TUSERDATA) { + bytes = fts_value_cost(fts_resolve_value(l, index, 0), 0); + if (bytes == SIZE_MAX) { + luaL_error(l, "typed value nesting exceeds limit"); + } + } + else if (lua_type(l, index) == LUA_TSTRING) { + lua_tolstring(l, index, &length); + bytes += length; + } + else if (lua_type(l, index) == LUA_TNUMBER) { + if (!isfinite(lua_tonumber(l, index))) { + luaL_error(l, "telemetry numbers must be finite"); + } + } + else if (lua_type(l, index) != LUA_TBOOLEAN) { + luaL_error(l, "use typed telemetry values and fb.null"); + } + + fts_charge_value(l, root_value(l, 1), cost_add(bytes, overhead)); +} + +static struct cfl_variant *from_lua(lua_State *l, int index, struct cfl_arena *arena) +{ + const char *str; + size_t length; + lua_Number number; + struct cfl_variant *v; + + switch (lua_type(l, index)) { + case LUA_TUSERDATA: + v = fts_copy_value_in(fts_resolve_value(l, index, 0), 0, arena); + break; + case LUA_TBOOLEAN: + v = cfl_variant_create_from_bool_in(arena, lua_toboolean(l, index)); + break; + case LUA_TSTRING: + str = lua_tolstring(l, index, &length); + v = cfl_variant_create_from_string_s_in(arena, (char *) str, length, CFL_FALSE); + break; + case LUA_TNUMBER: + number = lua_tonumber(l, index); + if (!isfinite(number)) { + luaL_error(l, "number must be finite"); + } + v = cfl_variant_create_from_double_in(arena, number); + break; + default: + luaL_error(l, "use a scalar or typed value; use fb.null instead of nil"); + return NULL; + } + + if (!v) { + luaL_error(l, "cannot copy typed value (allocation or nesting limit)"); + } + + return v; +} + +static void mark_dirty(lua_State *l, int index) +{ + struct fts_value *value; + + value = check_value(l, index); + if (!value->owned) { + value = value->root; + check_live(l, value); + } + + value->dirty = FLB_TRUE; +} + +/* Copy a borrowed root before its first write; views address it by path. */ +static void detach_root(lua_State *l, int index, int depth) +{ + struct fts_value *value; + struct cfl_variant *copy; + size_t cost; + + value = lua_touserdata(l, index); + if (!value->owned) { + if (depth >= FTS_MAX_DEPTH || !lua_checkstack(l, 4)) { + luaL_error(l, "typed value nesting exceeds limit"); + } + + lua_getfenv(l, index); + lua_rawgeti(l, -1, 1); + detach_root(l, lua_gettop(l), depth + 1); + lua_pop(l, 2); + return; + } + + if (!value->borrowed || value->inplace) { + return; + } + + check_live(l, value); + + /* + * Chunk trees are edited in place with an undo log. Written values are + * allocated from the container's own arena, if any. Shared empty maps + * are immutable. + */ + if (value->ctx->inplace_ok && value->owned != &empty_value) { + value->inplace = FLB_TRUE; + return; + } + + cost = fts_value_cost(value->owned, 0); + if (cost == SIZE_MAX) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + + /* The finalizer skips borrowed storage, so charges are released on failure. */ + lua_rawgeti(l, LUA_REGISTRYINDEX, value->ctx->value_metatable_ref); + lua_setmetatable(l, index); + if (value->charged == 0) { + fts_charge_value(l, value, cost); + } + + copy = fts_copy_value(value->owned, 0); + if (!copy) { + luaL_error(l, "cannot copy log value (allocation or nesting limit)"); + } + + value->owned = copy; + value->borrowed = FLB_FALSE; +} + +static struct cfl_variant *writable_value(lua_State *l) +{ + struct fts_value *value; + + value = check_value(l, 1); + if (value->readonly) { + luaL_error(l, "typed value is read-only"); + } + + detach_root(l, 1, 0); + (value->owned ? value : value->root)->epoch++; + + if (value->owned) { + value->dirty = FLB_TRUE; + return value->owned; + } + + mark_dirty(l, 1); + return fts_resolve_value(l, 1, 0); +} + +static int value_set(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_variant *replacement; + struct cfl_variant *old; + struct cfl_kvpair *pair; + struct cfl_kvpair *duplicate; + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_list *prev; + struct fts_value *self; + struct fts_value *root; + const char *key; + size_t length; + size_t position; + + v = writable_value(l); + root = root_value(l, 1); + if (root->inplace) { + undo_reserve(l, root->ctx); + } + + self = lua_touserdata(l, 1); + if (!self->owned || lua_type(l, 3) == LUA_TUSERDATA) { + root_value(l, 1)->depth_changed = FLB_TRUE; + } + + if (v->type == CFL_VARIANT_KVLIST) { + if (lua_type(l, 2) != LUA_TSTRING) { + return luaL_error(l, "map key must be a string"); + } + + key = lua_tolstring(l, 2, &length); + pair = fts_find_pair(v, key, length); + charge_mutation(l, 3, pair ? 0 : 128 + length); + replacement = from_lua(l, 3, container_arena(v)); + + if (pair) { + if (cfl_container_move_variant_to_kvlist(v->data.as_kvlist, replacement) != 0) { + cfl_variant_destroy(replacement); + return luaL_error(l, "cannot attach map value"); + } + + old = pair->val; + pair->val = replacement; + if (root->inplace) { + undo_add(root->ctx, UNDO_SET_PAIR, NULL, pair, NULL, NULL, 0, old); + } + else { + credit_value(l, root, fts_value_cost(old, 0)); + cfl_variant_destroy(old); + } + + cfl_list_foreach_safe(head, tmp, &v->data.as_kvlist->list) { + duplicate = cfl_list_entry(head, struct cfl_kvpair, _head); + if (duplicate != pair && cfl_sds_len(duplicate->key) == length && + memcmp(duplicate->key, key, length) == 0) { + if (root->inplace) { + undo_reserve(l, root->ctx); + prev = duplicate->_head.prev; + cfl_list_del(&duplicate->_head); + undo_add(root->ctx, UNDO_REMOVE_PAIR, &v->data.as_kvlist->list, + duplicate, prev, NULL, 0, NULL); + } + else { + credit_value(l, root, cost_add(128 + length, + fts_value_cost(duplicate->val, 0))); + cfl_kvpair_destroy(duplicate); + } + } + } + } + else if (cfl_kvlist_insert_s(v->data.as_kvlist, (char *) key, length, replacement) != 0) { + cfl_variant_destroy(replacement); + return luaL_error(l, "cannot insert map value"); + } + else if (root->inplace) { + undo_add(root->ctx, UNDO_INSERT_PAIR, NULL, + cfl_list_entry_last(&v->data.as_kvlist->list, struct cfl_kvpair, _head), + NULL, NULL, 0, NULL); + } + } + else if (v->type == CFL_VARIANT_ARRAY) { + position = array_index(l, v, 2); + charge_mutation(l, 3, 0); + replacement = from_lua(l, 3, container_arena(v)); + + if (cfl_container_move_variant_to_array(v->data.as_array, replacement) != 0) { + cfl_variant_destroy(replacement); + return luaL_error(l, "cannot attach array value"); + } + + old = v->data.as_array->entries[position]; + v->data.as_array->entries[position] = replacement; + if (root->inplace) { + undo_add(root->ctx, UNDO_SET_INDEX, NULL, NULL, NULL, v->data.as_array, position, old); + } + else { + credit_value(l, root, fts_value_cost(old, 0)); + cfl_variant_destroy(old); + } + } + else { + return luaL_error(l, "set() requires a map or array"); + } + + return 0; +} + +static int value_remove(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_kvpair *pair; + struct cfl_list *head; + struct cfl_list *next; + struct cfl_list *prev; + struct fts_value *owner; + const char *key; + size_t length; + int removed = FLB_FALSE; + + v = writable_value(l); + if (v->type != CFL_VARIANT_KVLIST || lua_type(l, 2) != LUA_TSTRING) { + return luaL_error(l, "remove() requires a map and string key"); + } + + key = lua_tolstring(l, 2, &length); + owner = root_value(l, 1); + + cfl_list_foreach_safe(head, next, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (cfl_sds_len(pair->key) == length && memcmp(pair->key, key, length) == 0) { + if (owner->inplace) { + undo_reserve(l, owner->ctx); + prev = pair->_head.prev; + cfl_list_del(&pair->_head); + undo_add(owner->ctx, UNDO_REMOVE_PAIR, &v->data.as_kvlist->list, + pair, prev, NULL, 0, NULL); + } + else { + credit_value(l, owner, cost_add(128 + length, fts_value_cost(pair->val, 0))); + cfl_kvpair_destroy(pair); + } + removed = FLB_TRUE; + } + } + + lua_pushboolean(l, removed); + return 1; +} + +static int value_ensure_map(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_variant *child; + struct cfl_kvlist *map; + struct cfl_kvpair *pair; + struct fts_value *root; + const char *key; + size_t length; + + v = fts_resolve_value(l, 1, 0); + if (v->type != CFL_VARIANT_KVLIST || lua_type(l, 2) != LUA_TSTRING) { + return luaL_error(l, "ensure_map() requires a map and string key"); + } + + key = lua_tolstring(l, 2, &length); + pair = fts_find_pair(v, key, length); + if (pair && pair->val->type != CFL_VARIANT_KVLIST) { + return luaL_error(l, "existing field is not a map"); + } + + if (!pair) { + v = writable_value(l); + root = root_value(l, 1); + root->depth_changed = FLB_TRUE; + if (root->inplace) { + undo_reserve(l, root->ctx); + } + + fts_charge_value(l, root, 384 + length); + + map = cfl_kvlist_create_in(container_arena(v)); + child = map ? cfl_variant_create_from_kvlist_in(container_arena(v), map) : NULL; + if (!child) { + cfl_kvlist_destroy(map); + return luaL_error(l, "cannot allocate map"); + } + + if (cfl_kvlist_insert_s(v->data.as_kvlist, (char *) key, length, child) != 0) { + cfl_variant_destroy(child); + return luaL_error(l, "cannot insert map"); + } + + if (root->inplace) { + undo_add(root->ctx, UNDO_INSERT_PAIR, NULL, + cfl_list_entry_last(&v->data.as_kvlist->list, struct cfl_kvpair, _head), + NULL, NULL, 0, NULL); + } + } + + push_view(l, 1, 2); + return 1; +} + +static int value_len(lua_State *l) +{ + struct cfl_variant *v; + + v = fts_resolve_value(l, 1, 0); + if (v->type == CFL_VARIANT_ARRAY) { + lua_pushnumber(l, v->data.as_array->entry_count); + } + else if (v->type == CFL_VARIANT_KVLIST) { + lua_pushnumber(l, cfl_kvlist_count(v->data.as_kvlist)); + } + else { + return luaL_error(l, "len() requires a map or array"); + } + + return 1; +} + +static int value_keys(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_list *head; + struct cfl_kvpair *pair; + int i = 1; + + v = fts_resolve_value(l, 1, 0); + if (v->type != CFL_VARIANT_KVLIST) { + return luaL_error(l, "keys() requires a map"); + } + + lua_newtable(l); + cfl_list_foreach(head, &v->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + lua_pushlstring(l, pair->key, cfl_sds_len(pair->key)); + lua_rawseti(l, -2, i++); + } + + return 1; +} + +static int value_append(lua_State *l) +{ + struct cfl_variant *v; + struct cfl_variant *child; + struct fts_value *root; + + v = writable_value(l); + root = root_value(l, 1); + root->depth_changed = FLB_TRUE; + if (v->type != CFL_VARIANT_ARRAY) { + return luaL_error(l, "append() requires an array"); + } + + if (root->inplace) { + undo_reserve(l, root->ctx); + } + + charge_mutation(l, 2, 0); + child = from_lua(l, 2, container_arena(v)); + cfl_array_resizable(v->data.as_array, CFL_TRUE); + if (cfl_array_append(v->data.as_array, child) != 0) { + cfl_variant_destroy(child); + return luaL_error(l, "cannot append array value"); + } + + if (root->inplace) { + undo_add(root->ctx, UNDO_APPEND, NULL, NULL, NULL, v->data.as_array, 0, NULL); + } + + return 0; +} + +static int new_integer(lua_State *l) +{ + const char *str; + char *end; + size_t length; + size_t i; + int is_unsigned; + int64_t signed_value; + uint64_t unsigned_value; + lua_Number number; + struct cfl_variant *v; + struct fts_value *owner; + + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, 256); + is_unsigned = lua_toboolean(l, lua_upvalueindex(1)); + + if (lua_type(l, 1) == LUA_TNUMBER) { + number = lua_tonumber(l, 1); + if (!isfinite(number) || number != floor(number) || fabs(number) > SAFE_INTEGER || + (is_unsigned && number < 0)) { + return luaL_error(l, "use a decimal string for integers outside the safe Lua range"); + } + + if (is_unsigned) { + v = cfl_variant_create_from_uint64((uint64_t) number); + } + else { + v = cfl_variant_create_from_int64((int64_t) number); + } + } + else { + str = luaL_checklstring(l, 1, &length); + i = (!is_unsigned && length > 0 && str[0] == '-') ? 1 : 0; + if (i == length) { + return luaL_error(l, "expected a decimal integer"); + } + + for (; i < length; i++) { + if (str[i] < '0' || str[i] > '9') { + return luaL_error(l, "expected a decimal integer"); + } + } + + errno = 0; + if (is_unsigned) { + unsigned_value = strtoull(str, &end, 10); + v = errno == ERANGE ? NULL : cfl_variant_create_from_uint64(unsigned_value); + } + else { + signed_value = strtoll(str, &end, 10); + v = errno == ERANGE ? NULL : cfl_variant_create_from_int64(signed_value); + } + + if (errno == ERANGE) { + return luaL_error(l, "integer outside 64-bit range"); + } + } + + if (!v) { + return luaL_error(l, "cannot allocate integer"); + } + + owner->owned = v; + return 1; +} + +static int new_container(lua_State *l) +{ + struct cfl_variant *v; + struct fts_value *owner; + struct cfl_kvlist *map; + struct cfl_array *array; + + if (lua_gettop(l) != 0) { + return luaL_error(l, "map()/array() create empty containers and take no arguments"); + } + + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, 256); + + if (lua_toboolean(l, lua_upvalueindex(1))) { + map = cfl_kvlist_create(); + v = map ? cfl_variant_create_from_kvlist(map) : NULL; + if (!v) { + cfl_kvlist_destroy(map); + } + } + else { + array = cfl_array_create(0); + v = array ? cfl_variant_create_from_array(array) : NULL; + if (!v) { + cfl_array_destroy(array); + } + } + + if (!v) { + return luaL_error(l, "cannot allocate container"); + } + + owner->owned = v; + return 1; +} + +static int new_bytes(lua_State *l) +{ + const char *str; + size_t length; + struct cfl_variant *v; + struct fts_value *owner; + + if (lua_type(l, 1) != LUA_TSTRING) { + return luaL_error(l, "bytes() requires a Lua string"); + } + + str = lua_tolstring(l, 1, &length); + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, length + 256); + + v = cfl_variant_create_from_bytes((char *) str, length, CFL_FALSE); + if (!v) { + return luaL_error(l, "cannot allocate bytes"); + } + + owner->owned = v; + return 1; +} + +static void push_folded_key(lua_State *l, const char *key, size_t length) +{ + luaL_Buffer buffer; + size_t index; + + luaL_buffinit(l, &buffer); + for (index = 0; index < length; index++) { + luaL_addchar(&buffer, tolower((unsigned char) key[index])); + } + luaL_pushresult(&buffer); +} + +/* Iterate the stored pairs, including duplicate parent keys. Named lookup + * cannot visit all occurrences and is unsuitable for privacy operations. */ +static void redact_value(lua_State *l, struct cfl_variant *value, int depth, int keys) +{ + struct cfl_list *head; + struct cfl_list *tmp; + struct cfl_kvpair *pair; + struct cfl_variant *replacement; + struct cfl_variant *old; + struct cfl_list *previous; + struct fts_runtime *ctx; + struct fts_value *owner; + void *data; + size_t length; + size_t i; + int match; + int seen; + int duplicate; + + lua_getallocf(l, &data); + ctx = data; + if (depth >= FTS_MAX_DEPTH || (!ctx->jit && --ctx->instructions_left <= 0)) { + luaL_error(l, "redaction nesting or instruction limit exceeded"); + } + + if (!lua_checkstack(l, 8)) { + luaL_error(l, "redaction stack limit exceeded"); + } + + owner = root_value(l, 1); + + if (value->type == CFL_VARIANT_ARRAY) { + for (i = 0; i < value->data.as_array->entry_count; i++) { + redact_value(l, value->data.as_array->entries[i], depth + 1, keys); + } + } + else if (value->type == CFL_VARIANT_KVLIST) { + lua_newtable(l); + seen = lua_gettop(l); + + cfl_list_foreach_safe(head, tmp, &value->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + length = cfl_sds_len(pair->key); + + push_folded_key(l, pair->key, length); + lua_rawget(l, keys); + match = lua_toboolean(l, -1); + lua_pop(l, 1); + + if (match) { + lua_pushlstring(l, pair->key, length); + lua_rawget(l, seen); + duplicate = lua_toboolean(l, -1); + lua_pop(l, 1); + + if (duplicate) { + if (owner->inplace) { + undo_reserve(l, owner->ctx); + previous = pair->_head.prev; + cfl_list_del(&pair->_head); + undo_add(owner->ctx, UNDO_REMOVE_PAIR, &value->data.as_kvlist->list, + pair, previous, NULL, 0, NULL); + } + else { + credit_value(l, owner, cost_add(128 + length, + fts_value_cost(pair->val, 0))); + cfl_kvpair_destroy(pair); + } + continue; + } + + lua_pushlstring(l, pair->key, length); + lua_pushboolean(l, 1); + lua_rawset(l, seen); + + if (owner->inplace) { + undo_reserve(l, owner->ctx); + } + + charge_mutation(l, 3, 0); + replacement = from_lua(l, 3, container_arena(value)); + if (cfl_container_move_variant_to_kvlist(value->data.as_kvlist, + replacement) != 0) { + cfl_variant_destroy(replacement); + luaL_error(l, "cannot attach redacted value"); + } + + old = pair->val; + pair->val = replacement; + if (owner->inplace) { + undo_add(owner->ctx, UNDO_SET_PAIR, NULL, pair, NULL, NULL, 0, old); + } + else { + credit_value(l, owner, fts_value_cost(old, 0)); + cfl_variant_destroy(old); + } + } + else { + redact_value(l, pair->val, depth + 1, keys); + } + } + + lua_pop(l, 1); + } +} + +static int redact(lua_State *l) +{ + struct cfl_variant *value; + const char *key; + size_t length; + size_t index; + size_t count; + int keys; + + luaL_checktype(l, 2, LUA_TTABLE); + if (lua_isnoneornil(l, 3)) { + lua_settop(l, 2); + lua_pushliteral(l, "[REDACTED]"); + } + + value = writable_value(l); + if (lua_type(l, 3) == LUA_TUSERDATA) { + root_value(l, 1)->depth_changed = FLB_TRUE; + } + + count = lua_objlen(l, 2); + lua_newtable(l); + keys = lua_gettop(l); + + for (index = 1; index <= count; index++) { + lua_rawgeti(l, 2, index); + luaL_checktype(l, -1, LUA_TSTRING); + key = lua_tolstring(l, -1, &length); + push_folded_key(l, key, length); + lua_pushboolean(l, 1); + lua_rawset(l, keys); + lua_pop(l, 1); + } + + redact_value(l, value, 0, keys); + return 0; +} + +/* An owned, writable copy that may outlive the callback. */ +static int value_copy(lua_State *l) +{ + struct cfl_variant *source; + struct fts_value *value; + size_t cost; + + source = fts_resolve_value(l, 1, 0); + cost = fts_value_cost(source, 0); + if (cost == SIZE_MAX) { + return luaL_error(l, "cannot copy typed value (allocation or nesting limit)"); + } + + value = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, value, cost); + + value->owned = fts_copy_value(source, 0); + if (!value->owned) { + return luaL_error(l, "cannot copy typed value (allocation or nesting limit)"); + } + + return 1; +} + +static const luaL_Reg value_methods[] = { + {"__gc", value_gc}, + {"type", value_type}, + {"value", value_scalar}, + {"number", value_number}, + {"get", value_get}, + {"get_string", value_get_string}, + {"get_bool", value_get_bool}, + {"get_number", value_get_number}, + {"set", value_set}, + {"remove", value_remove}, + {"ensure_map", value_ensure_map}, + {"len", value_len}, + {"keys", value_keys}, + {"append", value_append}, + {"copy", value_copy}, + {NULL, NULL} +}; + +static int json_key_compare(const void *a, const void *b) +{ + yyjson_val *left = *(yyjson_val * const *) a; + yyjson_val *right = *(yyjson_val * const *) b; + size_t left_length = yyjson_get_len(left); + size_t right_length = yyjson_get_len(right); + + if (left_length != right_length) { + return left_length < right_length ? -1 : 1; + } + + return memcmp(yyjson_get_str(left), yyjson_get_str(right), left_length); +} + +/* + * Duplicate keys are found by sorting the object keys: O(n log n) for any + * input, unlike hashing, which crafted colliding keys can make quadratic. + */ +static int json_object_has_duplicates(yyjson_val *value, struct fts_runtime *ctx) +{ + size_t i; + size_t index; + size_t count; + yyjson_val *key; + yyjson_val *item; + yyjson_val **keys; + int duplicate = FLB_FALSE; + + count = yyjson_obj_size(value); + if (count < 2) { + return FLB_FALSE; + } + + /* sorting is native work, charge it to the instruction budget */ + if (!ctx->jit) { + if ((size_t) ctx->instructions_left <= count) { + ctx->instructions_left = 0; + return FLB_TRUE; + } + ctx->instructions_left -= count; + } + + keys = flb_malloc(sizeof(yyjson_val *) * count); + if (!keys) { + flb_errno(); + return FLB_TRUE; + } + + i = 0; + yyjson_obj_foreach(value, index, count, key, item) { + (void) item; + keys[i++] = key; + } + + qsort(keys, count, sizeof(yyjson_val *), json_key_compare); + for (i = 1; i < count; i++) { + if (json_key_compare(&keys[i - 1], &keys[i]) == 0) { + duplicate = FLB_TRUE; + break; + } + } + + flb_free(keys); + return duplicate; +} + +static size_t json_value_cost(yyjson_val *value, int depth) +{ + yyjson_val *key; + yyjson_val *item; + size_t index; + size_t count; + size_t cost = 256; + size_t child; + + if (depth >= FTS_MAX_DEPTH) { + return SIZE_MAX; + } + + if (yyjson_is_str(value)) { + return cost + yyjson_get_len(value); + } + + if (yyjson_is_obj(value)) { + yyjson_obj_foreach(value, index, count, key, item) { + child = json_value_cost(item, depth + 1); + if (child > SIZE_MAX - cost || + yyjson_get_len(key) + 128 > SIZE_MAX - cost - child) { + return SIZE_MAX; + } + cost += child + yyjson_get_len(key) + 128; + } + } + else if (yyjson_is_arr(value)) { + yyjson_arr_foreach(value, index, count, item) { + child = json_value_cost(item, depth + 1); + if (child > SIZE_MAX - cost) { + return SIZE_MAX; + } + cost += child; + } + } + + return cost; +} + +/* Strict JSON conversion preserves integer widths and rejects duplicate keys. */ +static struct cfl_variant *json_value(yyjson_val *value, int depth, struct fts_runtime *ctx) +{ + struct cfl_variant *out; + struct cfl_variant *child; + struct cfl_array *array; + struct cfl_kvlist *map; + yyjson_val *key; + yyjson_val *item; + size_t index; + size_t count; + + if (depth >= FTS_MAX_DEPTH || (!ctx->jit && --ctx->instructions_left <= 0)) { + return NULL; + } + + if (yyjson_is_obj(value)) { + map = cfl_kvlist_create(); + out = map ? cfl_variant_create_from_kvlist(map) : NULL; + if (!out) { + cfl_kvlist_destroy(map); + return NULL; + } + + if (json_object_has_duplicates(value, ctx)) { + cfl_variant_destroy(out); + return NULL; + } + + yyjson_obj_foreach(value, index, count, key, item) { + child = json_value(item, depth + 1, ctx); + if (!child || cfl_kvlist_insert_s(map, (char *) yyjson_get_str(key), + yyjson_get_len(key), child) != 0) { + cfl_variant_destroy(child); + cfl_variant_destroy(out); + return NULL; + } + } + + return out; + } + + if (yyjson_is_arr(value)) { + array = cfl_array_create(yyjson_arr_size(value)); + out = array ? cfl_variant_create_from_array(array) : NULL; + if (!out) { + cfl_array_destroy(array); + return NULL; + } + + yyjson_arr_foreach(value, index, count, item) { + child = json_value(item, depth + 1, ctx); + if (!child || cfl_array_append(array, child) != 0) { + cfl_variant_destroy(child); + cfl_variant_destroy(out); + return NULL; + } + } + + return out; + } + + if (yyjson_is_str(value)) { + return cfl_variant_create_from_string_s((char *) yyjson_get_str(value), + yyjson_get_len(value), CFL_FALSE); + } + + if (yyjson_is_uint(value)) { + return cfl_variant_create_from_uint64(yyjson_get_uint(value)); + } + + if (yyjson_is_sint(value)) { + return cfl_variant_create_from_int64(yyjson_get_sint(value)); + } + + if (yyjson_is_real(value)) { + return cfl_variant_create_from_double(yyjson_get_real(value)); + } + + if (yyjson_is_bool(value)) { + return cfl_variant_create_from_bool(yyjson_get_bool(value)); + } + + if (yyjson_is_null(value)) { + return cfl_variant_create_from_null(); + } + + return NULL; +} + +static int parse_json(lua_State *l) +{ + struct fts_value *owner; + struct fts_runtime *ctx; + yyjson_doc *doc; + const char *text; + size_t length; + size_t reservation; + size_t cost; + + luaL_checktype(l, 1, LUA_TSTRING); + text = lua_tolstring(l, 1, &length); + if (length > 1048576) { + return luaL_error(l, "JSON input exceeds 1 MiB"); + } + + owner = fts_push_owned(l, NULL, FLB_FALSE); + ctx = owner->ctx; + + /* Bound the parser's temporary document separately from the owned value. */ + reservation = length * 16 + 256; + fts_charge_value(l, owner, reservation); + + doc = yyjson_read(text, length, YYJSON_READ_BIGNUM_AS_RAW); + if (!doc) { + return luaL_error(l, "invalid JSON"); + } + + cost = json_value_cost(yyjson_doc_get_root(doc), 0); + if (cost > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - cost || + ctx->lua_bytes > ctx->memory_limit - ctx->native_bytes - cost) { + lua_gc(l, LUA_GCCOLLECT, 0); + if (cost > ctx->memory_limit || ctx->native_bytes > ctx->memory_limit - cost || + ctx->lua_bytes > ctx->memory_limit - ctx->native_bytes - cost) { + yyjson_doc_free(doc); + return luaL_error(l, "JSON value exceeds policy memory or nesting limit"); + } + } + + fts_charge_value(l, owner, cost); + owner->owned = json_value(yyjson_doc_get_root(doc), 0, ctx); + yyjson_doc_free(doc); + credit_value(l, owner, reservation); + + if (!owner->owned) { + return luaL_error(l, + "JSON exceeds type/depth/instruction limits or contains duplicate keys"); + } + + return 1; +} + +/* Exact aggregation never rounds integer telemetry through lua_Number. */ +static int add_metric_values(lua_State *l) +{ + struct cfl_variant *left; + struct cfl_variant *right; + struct fts_value *owner; + int64_t a; + int64_t b; + uint64_t unsigned_value; + uint64_t magnitude; + uint64_t total; + double x; + double y; + + left = fts_resolve_value(l, 1, 0); + right = fts_resolve_value(l, 2, 0); + owner = fts_push_owned(l, NULL, FLB_FALSE); + fts_charge_value(l, owner, 256); + + if (left->type == CFL_VARIANT_INT && right->type == CFL_VARIANT_INT) { + a = left->data.as_int64; + b = right->data.as_int64; + if ((b > 0 && a > INT64_MAX - b) || (b < 0 && a < INT64_MIN - b)) { + return luaL_error(l, "metric integer aggregation overflow"); + } + owner->owned = cfl_variant_create_from_int64(a + b); + } + else if (left->type == CFL_VARIANT_UINT && right->type == CFL_VARIANT_UINT) { + if (left->data.as_uint64 > UINT64_MAX - right->data.as_uint64) { + return luaL_error(l, "metric count aggregation overflow"); + } + owner->owned = cfl_variant_create_from_uint64(left->data.as_uint64 + + right->data.as_uint64); + } + else if ((left->type == CFL_VARIANT_INT && right->type == CFL_VARIANT_UINT) || + (left->type == CFL_VARIANT_UINT && right->type == CFL_VARIANT_INT)) { + if (left->type == CFL_VARIANT_INT) { + a = left->data.as_int64; + } + else { + a = right->data.as_int64; + } + + if (left->type == CFL_VARIANT_UINT) { + unsigned_value = left->data.as_uint64; + } + else { + unsigned_value = right->data.as_uint64; + } + + if (a < 0) { + magnitude = (uint64_t) -(a + 1) + 1; + if (unsigned_value < magnitude) { + magnitude -= unsigned_value; + if (magnitude == (uint64_t) INT64_MAX + 1) { + owner->owned = cfl_variant_create_from_int64(INT64_MIN); + } + else { + owner->owned = cfl_variant_create_from_int64(-(int64_t) magnitude); + } + } + else { + total = unsigned_value - magnitude; + if (total <= INT64_MAX) { + owner->owned = cfl_variant_create_from_int64((int64_t) total); + } + else { + owner->owned = cfl_variant_create_from_uint64(total); + } + } + } + else { + if (unsigned_value > UINT64_MAX - (uint64_t) a) { + return luaL_error(l, "metric integer aggregation overflow"); + } + + total = unsigned_value + (uint64_t) a; + if (total <= INT64_MAX) { + owner->owned = cfl_variant_create_from_int64((int64_t) total); + } + else { + owner->owned = cfl_variant_create_from_uint64(total); + } + } + } + else { + if ((left->type != CFL_VARIANT_INT && left->type != CFL_VARIANT_UINT && + left->type != CFL_VARIANT_DOUBLE) || + (right->type != CFL_VARIANT_INT && right->type != CFL_VARIANT_UINT && + right->type != CFL_VARIANT_DOUBLE)) { + return luaL_error(l, "metric aggregation requires numeric values"); + } + + if (left->type == CFL_VARIANT_DOUBLE) { + x = left->data.as_double; + } + else if (left->type == CFL_VARIANT_INT) { + x = (double) left->data.as_int64; + } + else { + x = (double) left->data.as_uint64; + } + + if (right->type == CFL_VARIANT_DOUBLE) { + y = right->data.as_double; + } + else if (right->type == CFL_VARIANT_INT) { + y = (double) right->data.as_int64; + } + else { + y = (double) right->data.as_uint64; + } + + if ((left->type != CFL_VARIANT_DOUBLE && fabs(x) > SAFE_INTEGER) || + (right->type != CFL_VARIANT_DOUBLE && fabs(y) > SAFE_INTEGER) || !isfinite(x + y)) { + return luaL_error(l, "metric aggregation would lose precision or overflow"); + } + owner->owned = cfl_variant_create_from_double(x + y); + } + + if (!owner->owned) { + return luaL_error(l, "cannot allocate aggregated value"); + } + + return 1; +} + +int fts_register_values(lua_State *l) +{ + struct fts_value *null_owner; + struct fts_runtime *ctx; + int index; + static const char *keys[] = {"tag", "body", "metadata", "timestamp", "sec", "nsec", + "context", "attributes"}; + + lua_getallocf(l, (void **) &ctx); + for (index = 0; index < FTS_KEY_COUNT; index++) { + lua_pushstring(l, keys[index]); + ctx->key_refs[index] = luaL_ref(l, LUA_REGISTRYINDEX); + } + + lua_pushliteral(l, ""); + ctx->tag_ref = luaL_ref(l, LUA_REGISTRYINDEX); + + /* Value metatable: finalizer plus the shared method table. */ + luaL_newmetatable(l, VALUE_MT); + ctx->value_metatable = lua_topointer(l, -1); + lua_pushvalue(l, -1); + ctx->value_metatable_ref = luaL_ref(l, LUA_REGISTRYINDEX); + luaL_register(l, NULL, value_methods); + lua_newtable(l); + luaL_register(l, NULL, value_methods + 1); + lua_setfield(l, -2, "__index"); + lua_pushliteral(l, "typed value"); + lua_setfield(l, -2, "__metatable"); + + /* Finalizer-free metatable for borrowed values and views. */ + lua_newtable(l); + lua_getfield(l, -2, "__index"); + lua_setfield(l, -2, "__index"); + lua_pushliteral(l, "typed value"); + lua_setfield(l, -2, "__metatable"); + ctx->empty_metatable = lua_topointer(l, -1); + ctx->empty_metatable_ref = luaL_ref(l, LUA_REGISTRYINDEX); + lua_pop(l, 1); + + /* package.loaded.fluentbit */ + lua_getglobal(l, "package"); + lua_getfield(l, -1, "loaded"); + lua_newtable(l); + + lua_pushboolean(l, 0); + lua_pushcclosure(l, new_integer, 1); + lua_setfield(l, -2, "int64"); + + lua_pushboolean(l, 1); + lua_pushcclosure(l, new_integer, 1); + lua_setfield(l, -2, "uint64"); + + lua_pushboolean(l, 1); + lua_pushcclosure(l, new_container, 1); + lua_setfield(l, -2, "map"); + + lua_pushboolean(l, 0); + lua_pushcclosure(l, new_container, 1); + lua_setfield(l, -2, "array"); + + lua_pushcfunction(l, equal_values); + lua_setfield(l, -2, "_fts_equal"); + lua_pushcfunction(l, add_metric_values); + lua_setfield(l, -2, "_fts_add"); + lua_pushcfunction(l, redact); + lua_setfield(l, -2, "redact"); + lua_pushcfunction(l, parse_json); + lua_setfield(l, -2, "parse_json"); + lua_pushcfunction(l, new_bytes); + lua_setfield(l, -2, "bytes"); + + null_owner = fts_push_owned(l, NULL, FLB_TRUE); + fts_charge_value(l, null_owner, 256); + null_owner->owned = cfl_variant_create_from_null(); + if (!null_owner->owned) { + lua_pop(l, 3); + return -1; + } + lua_setfield(l, -2, "null"); + + lua_pushliteral(l, "keep"); + lua_setfield(l, -2, "KEEP"); + lua_pushliteral(l, "drop"); + lua_setfield(l, -2, "DROP"); + + lua_setfield(l, -2, "fluentbit"); + lua_pop(l, 2); + + if (fts_load_helpers(l, fts_value_helpers, sizeof(fts_value_helpers) - 1, + "fluentbit.helpers") != 0) { + return -1; + } + + return 0; +} + +int fts_valid_utf8(const char *text, size_t length) +{ + const unsigned char *bytes = (const unsigned char *) text; + size_t i = 0; + unsigned char first; + unsigned char second; + int count; + int j; + + while (i < length) { + first = bytes[i++]; + if (first < 0x80) { + continue; + } + + if (first >= 0xc2 && first <= 0xdf) { + count = 1; + } + else if (first >= 0xe0 && first <= 0xef) { + count = 2; + } + else if (first >= 0xf0 && first <= 0xf4) { + count = 3; + } + else { + count = 0; + } + + if (!count || length - i < (size_t) count) { + return FLB_FALSE; + } + + second = bytes[i]; + if ((first == 0xe0 && second < 0xa0) || (first == 0xed && second > 0x9f) || + (first == 0xf0 && second < 0x90) || (first == 0xf4 && second > 0x8f)) { + return FLB_FALSE; + } + + for (j = 0; j < count; j++) { + if (bytes[i] < 0x80 || bytes[i++] > 0xbf) { + return FLB_FALSE; + } + } + } + + return FLB_TRUE; +} diff --git a/plugins/processor_labels/labels.c b/plugins/processor_labels/labels.c index 7f2e8aac940..f81efc3be07 100644 --- a/plugins/processor_labels/labels.c +++ b/plugins/processor_labels/labels.c @@ -196,6 +196,7 @@ static void map_metric_destroy(struct cmt_metric *metric) free(metric->sum_quantiles); } + cfl_kvlist_destroy(metric->otlp_metadata); cfl_list_del(&metric->_head); free(metric); } @@ -806,6 +807,7 @@ int metrics_map_convert_static_metric(struct cmt_map *map, char *label_value) { struct cmt_metric *metric; + struct cfl_list labels; int result; size_t index; cfl_hash_state_t state; @@ -861,23 +863,22 @@ int metrics_map_convert_static_metric(struct cmt_map *map, } } - metric->val = map->metric.val; - - metric->hist_buckets = map->metric.hist_buckets; - metric->hist_count = map->metric.hist_count; - metric->hist_sum = map->metric.hist_sum; - - metric->sum_quantiles_set = map->metric.sum_quantiles_set; - metric->sum_quantiles = map->metric.sum_quantiles; - metric->sum_quantiles_count = map->metric.sum_quantiles_count; - metric->sum_count = map->metric.sum_count; - metric->sum_sum = map->metric.sum_sum; - - metric->timestamp = map->metric.timestamp; - - map->metric_static_set = 0; + /* Transfer the complete sample, including exact scalar types, start time, + * distributions and owned metadata. Preserve the newly allocated labels. */ + labels = metric->labels; + memcpy(metric, &map->metric, sizeof(*metric)); + metric->labels = labels; + metric->map = map; + metric->hash = hash; + metric->hash_indexed = CMT_FALSE; + cfl_list_init(&metric->_hash_head); + map->metric_static_set = CMT_FALSE; - cfl_list_add(&metric->_head, &map->metrics); + /* The static sample preceded dynamic samples before the conversion. */ + cfl_list_add_before(&metric->_head, map->metrics.next, &map->metrics); + if (map->last_metric == &map->metric) { + map->last_metric = metric; + } memset(&map->metric, 0, sizeof(struct cmt_metric)); diff --git a/plugins/processor_opentelemetry_envelope/otel_envelope.c b/plugins/processor_opentelemetry_envelope/otel_envelope.c index 9978318e95a..d6bd625df08 100644 --- a/plugins/processor_opentelemetry_envelope/otel_envelope.c +++ b/plugins/processor_opentelemetry_envelope/otel_envelope.c @@ -159,8 +159,8 @@ static void envelop_end(struct cfl_list *list, struct flb_mp_chunk_record *activ record->cobj_metadata = NULL; record->cobj_record = NULL; - /* add the envelop before the active record */ - cfl_list_add_after(&record->_head, &active_record->_head, list); + /* Insert after the active record while updating both list directions. */ + cfl_list_add_before(&record->_head, active_record->_head.next, list); } diff --git a/src/flb_input_log.c b/src/flb_input_log.c index e14c84820c1..764c9893fc1 100644 --- a/src/flb_input_log.c +++ b/src/flb_input_log.c @@ -1526,13 +1526,20 @@ static int input_log_append_processed_internal(struct flb_input_instance *ins, return ret; } +/* + * 'exact_records' tells whether 'records' is the exact number of log records in + * the buffer. Caller-provided hints (for example, lines read by an input) are + * not forwarded to processors, which count the records themselves. + */ static int input_log_append(struct flb_input_instance *ins, size_t processor_starting_stage, - size_t records, + size_t records, int exact_records, const char *tag, size_t tag_len, const void *buf, size_t buf_size) { int ret; + int known_records = -1; + int processed_records = -1; int processor_is_active; void *out_buf = (void *) buf; size_t out_size = buf_size; @@ -1550,12 +1557,17 @@ static int input_log_append(struct flb_input_instance *ins, } } - ret = flb_processor_run(ins->processor, - processor_starting_stage, - FLB_PROCESSOR_LOGS, - tag, tag_len, - (char *) buf, buf_size, - &out_buf, &out_size); + if (exact_records && records <= INT_MAX) { + known_records = (int) records; + } + + ret = flb_processor_run_with_count(ins->processor, + processor_starting_stage, + FLB_PROCESSOR_LOGS, + tag, tag_len, + (char *) buf, buf_size, + &out_buf, &out_size, + known_records, &processed_records); if (ret == -1) { return -1; } @@ -1567,9 +1579,13 @@ static int input_log_append(struct flb_input_instance *ins, return 0; } - if (buf != out_buf) { - /* a new buffer was created, re-count the number of records */ - records = flb_mp_count_log_records(out_buf, out_size); + if (buf != out_buf || exact_records) { + if (processed_records >= 0) { + records = processed_records; + } + else if (buf != out_buf) { + records = flb_mp_count_log_records(out_buf, out_size); + } } } @@ -1604,7 +1620,7 @@ int flb_input_log_append(struct flb_input_instance *ins, size_t records; records = flb_mp_count_log_records(buf, buf_size); - ret = input_log_append(ins, 0, records, tag, tag_len, buf, buf_size); + ret = input_log_append(ins, 0, records, FLB_TRUE, tag, tag_len, buf, buf_size); return ret; } @@ -1636,6 +1652,7 @@ int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins, return input_log_append(ins, processor_starting_stage, flb_mp_count_log_records(buf, buf_size), + FLB_TRUE, tag, tag_len, buf, @@ -1650,6 +1667,6 @@ int flb_input_log_append_records(struct flb_input_instance *ins, { int ret; - ret = input_log_append(ins, 0, records, tag, tag_len, buf, buf_size); + ret = input_log_append(ins, 0, records, FLB_FALSE, tag, tag_len, buf, buf_size); return ret; } diff --git a/src/flb_json.c b/src/flb_json.c index 7bf452fe521..3e8e1b5cb65 100644 --- a/src/flb_json.c +++ b/src/flb_json.c @@ -511,62 +511,18 @@ static yyjson_mut_val *msgpack_to_yyjson_mut(yyjson_mut_doc *document, */ #define FLB_JSON_MUT_RENDER_MAX_DEPTH 1024 -static yyjson_mut_val *mutable_to_yyjson_mut_internal(yyjson_mut_doc *document, - struct flb_json_mut_val *value, - size_t depth) +static yyjson_mut_val *mutable_value_to_yyjson(yyjson_mut_doc *document, + struct flb_json_mut_val *value) { - struct flb_json_mut_kv *kv_entry; - struct flb_json_mut_entry *array_entry; - yyjson_mut_val *result; - yyjson_mut_val *item; - - if (depth > FLB_JSON_MUT_RENDER_MAX_DEPTH) { + if (value == NULL) { return NULL; } switch (value->type) { case FLB_JSON_MUT_OBJECT: - result = yyjson_mut_obj(document); - if (result == NULL) { - return NULL; - } - - for (kv_entry = value->data.object.head; - kv_entry != NULL; - kv_entry = kv_entry->next) { - item = mutable_to_yyjson_mut_internal(document, - kv_entry->value, - depth + 1); - if (item == NULL) { - return NULL; - } - - if (!yyjson_mut_obj_add(result, - yyjson_mut_strcpy(document, kv_entry->key), - item)) { - return NULL; - } - } - - return result; + return yyjson_mut_obj(document); case FLB_JSON_MUT_ARRAY: - result = yyjson_mut_arr(document); - if (result == NULL) { - return NULL; - } - - for (array_entry = value->data.array.head; - array_entry != NULL; - array_entry = array_entry->next) { - item = mutable_to_yyjson_mut_internal(document, - array_entry->value, - depth + 1); - if (item == NULL || !yyjson_mut_arr_add_val(result, item)) { - return NULL; - } - } - - return result; + return yyjson_mut_arr(document); case FLB_JSON_MUT_STRING: return yyjson_mut_strcpy(document, value->data.string); case FLB_JSON_MUT_BOOL: @@ -584,10 +540,113 @@ static yyjson_mut_val *mutable_to_yyjson_mut_internal(yyjson_mut_doc *document, } } +/* + * Output workers run on small coroutine stacks, so the traversal keeps its own + * stack on the heap: even modest OTLP nesting expands into many JSON object and + * array layers. + */ static yyjson_mut_val *mutable_to_yyjson_mut(yyjson_mut_doc *document, struct flb_json_mut_val *value) { - return mutable_to_yyjson_mut_internal(document, value, 0); + struct render_frame { + yyjson_mut_val *output; + struct flb_json_mut_kv *object; + struct flb_json_mut_entry *array; + }; + int ret; + struct render_frame *frames; + struct render_frame *frame; + struct flb_json_mut_val *child; + yyjson_mut_val *result; + yyjson_mut_val *item; + const char *key; + size_t depth; + + result = mutable_value_to_yyjson(document, value); + if (result == NULL) { + return NULL; + } + + frames = flb_calloc(FLB_JSON_MUT_RENDER_MAX_DEPTH + 1, sizeof(*frames)); + if (frames == NULL) { + flb_errno(); + return NULL; + } + + /* the root frame iterates the children of the top level container */ + depth = 0; + frames[0].output = result; + if (value->type == FLB_JSON_MUT_OBJECT) { + frames[0].object = value->data.object.head; + } + else if (value->type == FLB_JSON_MUT_ARRAY) { + frames[0].array = value->data.array.head; + } + + while (1) { + frame = &frames[depth]; + key = NULL; + + /* take the next child of the current container, or pop the frame */ + if (frame->object != NULL) { + key = frame->object->key; + child = frame->object->value; + frame->object = frame->object->next; + } + else if (frame->array != NULL) { + child = frame->array->value; + frame->array = frame->array->next; + } + else { + if (depth == 0) { + break; + } + depth--; + continue; + } + + item = mutable_value_to_yyjson(document, child); + if (item == NULL) { + result = NULL; + break; + } + + if (key) { + ret = yyjson_mut_obj_add(frame->output, yyjson_mut_strcpy(document, key), item); + } + else { + ret = yyjson_mut_arr_add_val(frame->output, item); + } + + if (!ret) { + result = NULL; + break; + } + + /* containers get their own frame so their children are visited next */ + if (child->type == FLB_JSON_MUT_OBJECT || child->type == FLB_JSON_MUT_ARRAY) { + if (depth == FLB_JSON_MUT_RENDER_MAX_DEPTH) { + result = NULL; + break; + } + + frame = &frames[++depth]; + frame->output = item; + frame->object = NULL; + frame->array = NULL; + + if (child->type == FLB_JSON_MUT_OBJECT) { + frame->object = child->data.object.head; + } + else { + frame->array = child->data.array.head; + } + } + } + + flb_free(frames); + + return result; } static char *render_msgpack_document_yyjson(struct flb_json_doc *document, diff --git a/src/flb_log_event_decoder.c b/src/flb_log_event_decoder.c index fa96eb07842..cfd8920fe00 100644 --- a/src/flb_log_event_decoder.c +++ b/src/flb_log_event_decoder.c @@ -370,10 +370,31 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, context->record_length = 0; previous_offset = context->offset; - result = msgpack_unpack_next(&context->unpacked_event, - context->buffer, - context->length, - &context->offset); + + /* + * Reuse one zone across records. Clearing keeps its first chunk, whereas + * msgpack_unpack_next() frees and recreates the zone for every record. A + * group start takes the zone over, so a new one is created on the next call. + */ + if (context->unpacked_event.zone == NULL) { + context->unpacked_event.zone = msgpack_zone_new(MSGPACK_ZONE_CHUNK_SIZE); + if (context->unpacked_event.zone == NULL) { + context->last_result = FLB_EVENT_DECODER_ERROR_DESERIALIZATION_FAILURE; + return context->last_result; + } + } + else { + msgpack_zone_clear(context->unpacked_event.zone); + } + + memset(&context->unpacked_event.data, 0, sizeof(msgpack_object)); + result = msgpack_unpack(context->buffer, context->length, &context->offset, + context->unpacked_event.zone, &context->unpacked_event.data); + + /* remaining bytes belong to the next records */ + if (result == MSGPACK_UNPACK_EXTRA_BYTES) { + result = MSGPACK_UNPACK_SUCCESS; + } if (result == MSGPACK_UNPACK_CONTINUE) { context->last_result = FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA; @@ -460,9 +481,8 @@ int flb_log_event_decoder_next(struct flb_log_event_decoder *context, if (context->read_groups != FLB_TRUE) { /* - * Skip group markers and move to the next record. - * msgpack_unpack_next will properly destroy and reinitialize - * unpacked_event, so no explicit cleanup needed here. + * Skip group markers and move to the next record. The next + * iteration clears or recreates the unpacked_event zone. */ memset(event, 0, sizeof(struct flb_log_event)); goto next_record; diff --git a/src/flb_mp.c b/src/flb_mp.c index d17f8c04c30..d02e91281da 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -46,6 +47,185 @@ int flb_mp_count(const void *data, size_t bytes) return flb_mp_count_remaining(data, bytes, NULL); } +/* + * Validate a MessagePack value with a read-only traversal: no msgpack object + * tree or zone is constructed. Unusual or deeper inputs are rejected so the + * counting API can fall back to the event decoder. + */ +static int scan_log_value(mpack_reader_t *reader, mpack_tag_t tag, int depth) +{ + int ret; + uint32_t index; + uint32_t count; + mpack_tag_t key; + + if (depth >= 64 || mpack_reader_error(reader) != mpack_ok) { + return -1; + } + + switch (mpack_tag_type(&tag)) { + case mpack_type_array: + count = mpack_tag_array_count(&tag); + for (index = 0; index < count; index++) { + ret = scan_log_value(reader, mpack_read_tag(reader), depth + 1); + if (ret != 0) { + return -1; + } + } + mpack_done_array(reader); + break; + case mpack_type_map: + count = mpack_tag_map_count(&tag); + for (index = 0; index < count; index++) { + key = mpack_read_tag(reader); + if (scan_log_value(reader, key, depth + 1) != 0) { + return -1; + } + + ret = scan_log_value(reader, mpack_read_tag(reader), depth + 1); + if (ret != 0) { + return -1; + } + } + mpack_done_map(reader); + break; + case mpack_type_str: + mpack_skip_bytes(reader, mpack_tag_str_length(&tag)); + mpack_done_str(reader); + break; + case mpack_type_bin: + mpack_skip_bytes(reader, mpack_tag_bin_length(&tag)); + mpack_done_bin(reader); + break; + case mpack_type_ext: + mpack_skip_bytes(reader, mpack_tag_ext_length(&tag)); + mpack_done_ext(reader); + break; + default: + break; + } + + if (mpack_reader_error(reader) != mpack_ok) { + return -1; + } + + return 0; +} + +int flb_mp_scan_log_records(const void *data, size_t bytes, int *records) +{ + mpack_reader_t reader; + mpack_tag_t tag; + msgpack_object timestamp = {0}; + struct flb_time time; + char extension[8]; + int header; + int count = 0; + int ret = -1; + + /* mpack asserts on a NULL buffer even when it is empty */ + if (!data) { + if (bytes) { + return -1; + } + *records = 0; + return 0; + } + + mpack_reader_init_data(&reader, data, bytes); + + while (mpack_reader_remaining(&reader, NULL) > 0) { + /* every event is a two entries array: [header|timestamp, body] */ + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_array || mpack_tag_array_count(&tag) != 2) { + goto done; + } + + /* the header form is [timestamp, metadata] */ + tag = mpack_read_tag(&reader); + header = mpack_tag_type(&tag) == mpack_type_array; + if (header) { + if (mpack_tag_array_count(&tag) != 2) { + goto done; + } + tag = mpack_read_tag(&reader); + } + + if (mpack_tag_type(&tag) == mpack_type_uint) { + timestamp.type = MSGPACK_OBJECT_POSITIVE_INTEGER; + timestamp.via.u64 = mpack_tag_uint_value(&tag); + } + else if (mpack_tag_type(&tag) == mpack_type_ext && + mpack_tag_ext_length(&tag) == 8 && mpack_tag_ext_exttype(&tag) == 0) { + mpack_read_bytes(&reader, extension, sizeof(extension)); + mpack_done_ext(&reader); + + timestamp.type = MSGPACK_OBJECT_EXT; + timestamp.via.ext.ptr = extension; + timestamp.via.ext.size = sizeof(extension); + timestamp.via.ext.type = 0; + } + else { + /* the legacy decoder remains authoritative for float/int timestamps */ + goto done; + } + + if (mpack_reader_error(&reader) != mpack_ok || + flb_log_event_decoder_decode_timestamp(×tamp, &time) != 0) { + goto done; + } + + /* metadata */ + if (header) { + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_map || + scan_log_value(&reader, tag, 0) != 0) { + goto done; + } + mpack_done_array(&reader); + } + + /* body */ + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_map || + scan_log_value(&reader, tag, 0) != 0) { + goto done; + } + mpack_done_array(&reader); + + if (mpack_reader_error(&reader) != mpack_ok) { + goto done; + } + + /* group markers are not counted as records */ + if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_START || + time.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { + continue; + } + else if (flb_time_is_valid_eventtime(&time) == FLB_TRUE && count < INT_MAX) { + count++; + } + else { + goto done; + } + } + + *records = count; + ret = 0; + +done: + /* + * Flag an error on early exits so the reader does not complain about + * unfinished compound types when it is destroyed. + */ + if (ret != 0) { + mpack_reader_flag_error(&reader, mpack_error_invalid); + } + mpack_reader_destroy(&reader); + + return ret; +} + /* Return the number of log events excluding group markers */ int flb_mp_count_log_records(const void *data, size_t bytes) { @@ -54,6 +234,10 @@ int flb_mp_count_log_records(const void *data, size_t bytes) struct flb_log_event log_event; struct flb_log_event_decoder decoder; + if (flb_mp_scan_log_records(data, bytes, &count) == 0) { + return count; + } + count = 0; ret = flb_log_event_decoder_init(&decoder, (char *) data, bytes); @@ -71,6 +255,298 @@ int flb_mp_count_log_records(const void *data, size_t bytes) return count; } +/* + * Report the group markers of a log buffer that belong to neighbouring + * appends: an end marker before any start (FLB_MP_GROUPS_KEEP_LEADING_END) and + * a single group still open at the end (FLB_MP_GROUPS_KEEP_TRAILING_OPEN). + * Returns the flags, or 0 when the buffer cannot be scanned. + */ +static int log_group_scan(const void *data, size_t bytes, int *starts); + +int flb_mp_log_group_boundaries(const void *data, size_t bytes) +{ + return log_group_scan(data, bytes, NULL); +} + +/* number of group start markers in a log buffer, 0 when it cannot be scanned */ +int flb_mp_log_group_starts(const void *data, size_t bytes) +{ + int starts = 0; + + log_group_scan(data, bytes, &starts); + return starts; +} + +static int log_group_scan(const void *data, size_t bytes, int *starts) +{ + mpack_reader_t reader; + mpack_tag_t tag; + msgpack_object timestamp = {0}; + struct flb_time time; + char extension[8]; + int header; + int flags = 0; + int depth = 0; + int seen_start = FLB_FALSE; + int failed = FLB_TRUE; + + if (!data || bytes == 0) { + return 0; + } + + mpack_reader_init_data(&reader, data, bytes); + + while (mpack_reader_remaining(&reader, NULL) > 0) { + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_array || mpack_tag_array_count(&tag) != 2) { + goto done; + } + + tag = mpack_read_tag(&reader); + header = mpack_tag_type(&tag) == mpack_type_array; + if (header) { + if (mpack_tag_array_count(&tag) != 2) { + goto done; + } + tag = mpack_read_tag(&reader); + } + + /* only group markers matter, and they use the EventTime extension */ + timestamp.type = MSGPACK_OBJECT_POSITIVE_INTEGER; + if (mpack_tag_type(&tag) == mpack_type_ext && + mpack_tag_ext_length(&tag) == 8 && mpack_tag_ext_exttype(&tag) == 0) { + mpack_read_bytes(&reader, extension, sizeof(extension)); + mpack_done_ext(&reader); + timestamp.type = MSGPACK_OBJECT_EXT; + timestamp.via.ext.ptr = extension; + timestamp.via.ext.size = sizeof(extension); + timestamp.via.ext.type = 0; + } + else if (mpack_tag_type(&tag) == mpack_type_ext) { + mpack_skip_bytes(&reader, mpack_tag_ext_length(&tag)); + mpack_done_ext(&reader); + } + else if (mpack_tag_type(&tag) == mpack_type_array || + mpack_tag_type(&tag) == mpack_type_map || + mpack_tag_type(&tag) == mpack_type_str || + mpack_tag_type(&tag) == mpack_type_bin) { + goto done; + } + + if (header) { + mpack_discard(&reader); + mpack_done_array(&reader); + } + mpack_discard(&reader); + mpack_done_array(&reader); + + if (mpack_reader_error(&reader) != mpack_ok) { + goto done; + } + + if (timestamp.type != MSGPACK_OBJECT_EXT || + flb_log_event_decoder_decode_timestamp(×tamp, &time) != 0) { + continue; + } + + if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_START) { + seen_start = FLB_TRUE; + depth++; + if (starts) { + (*starts)++; + } + } + else if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { + if (depth > 0) { + depth--; + } + else if (!seen_start) { + flags |= FLB_MP_GROUPS_KEEP_LEADING_END; + } + } + } + + if (depth == 1) { + flags |= FLB_MP_GROUPS_KEEP_TRAILING_OPEN; + } + failed = FLB_FALSE; + +done: + if (failed) { + mpack_reader_flag_error(&reader, mpack_error_invalid); + } + mpack_reader_destroy(&reader); + + return failed ? 0 : flags; +} + +/* TRUE when the event starting at 'data' is a group end marker */ +static int log_event_is_group_end(const char *data, size_t bytes) +{ + mpack_reader_t reader; + mpack_tag_t tag; + msgpack_object timestamp = {0}; + struct flb_time time; + char extension[8]; + int result = FLB_FALSE; + + mpack_reader_init_data(&reader, data, bytes); + + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) == mpack_type_array && mpack_tag_array_count(&tag) == 2) { + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) == mpack_type_array && mpack_tag_array_count(&tag) == 2) { + tag = mpack_read_tag(&reader); + } + + /* group markers always use the EventTime extension */ + if (mpack_tag_type(&tag) == mpack_type_ext && + mpack_tag_ext_length(&tag) == 8 && mpack_tag_ext_exttype(&tag) == 0) { + mpack_read_bytes(&reader, extension, sizeof(extension)); + if (mpack_reader_error(&reader) == mpack_ok) { + timestamp.type = MSGPACK_OBJECT_EXT; + timestamp.via.ext.ptr = extension; + timestamp.via.ext.size = sizeof(extension); + timestamp.via.ext.type = 0; + result = flb_log_event_decoder_decode_timestamp(×tamp, &time) == 0 && + time.tm.tv_sec == FLB_LOG_EVENT_GROUP_END; + } + } + } + + /* only the event header was read */ + mpack_reader_flag_error(&reader, mpack_error_invalid); + mpack_reader_destroy(&reader); + + return result; +} + +/* + * Find the end of the log window starting at 'offset'. A window closes after + * 'limit_bytes' of serialized events or 'limit_records' log records, but only + * outside a group: every group stays whole within one window, which can exceed + * the limits by up to one group. + */ +int flb_mp_log_window_next(const void *data, size_t bytes, size_t offset, + size_t limit_bytes, int limit_records, + size_t *window_end, int *window_records) +{ + mpack_reader_t reader; + mpack_tag_t tag; + msgpack_object timestamp = {0}; + struct flb_time time; + char extension[8]; + size_t position = 0; + int header; + int records = 0; + int in_group = FLB_FALSE; + int ret = -1; + + if (!data || !window_end || !window_records || offset >= bytes) { + return -1; + } + + mpack_reader_init_data(&reader, (const char *) data + offset, bytes - offset); + + while (mpack_reader_remaining(&reader, NULL) > 0) { + /* every event is a two entries array: [header|timestamp, body] */ + tag = mpack_read_tag(&reader); + if (mpack_tag_type(&tag) != mpack_type_array || mpack_tag_array_count(&tag) != 2) { + goto done; + } + + /* the header form is [timestamp, metadata] */ + tag = mpack_read_tag(&reader); + header = mpack_tag_type(&tag) == mpack_type_array; + if (header) { + if (mpack_tag_array_count(&tag) != 2) { + goto done; + } + tag = mpack_read_tag(&reader); + } + + if (mpack_tag_type(&tag) == mpack_type_uint) { + timestamp.type = MSGPACK_OBJECT_POSITIVE_INTEGER; + timestamp.via.u64 = mpack_tag_uint_value(&tag); + } + else if (mpack_tag_type(&tag) == mpack_type_double) { + timestamp.type = MSGPACK_OBJECT_FLOAT; + timestamp.via.f64 = mpack_tag_double_value(&tag); + } + else if (mpack_tag_type(&tag) == mpack_type_float) { + timestamp.type = MSGPACK_OBJECT_FLOAT; + timestamp.via.f64 = mpack_tag_float_value(&tag); + } + else if (mpack_tag_type(&tag) == mpack_type_ext && + mpack_tag_ext_length(&tag) == 8 && mpack_tag_ext_exttype(&tag) == 0) { + mpack_read_bytes(&reader, extension, sizeof(extension)); + mpack_done_ext(&reader); + + timestamp.type = MSGPACK_OBJECT_EXT; + timestamp.via.ext.ptr = extension; + timestamp.via.ext.size = sizeof(extension); + timestamp.via.ext.type = 0; + } + else { + goto done; + } + + if (header) { + mpack_discard(&reader); + mpack_done_array(&reader); + } + + /* body */ + mpack_discard(&reader); + mpack_done_array(&reader); + + if (mpack_reader_error(&reader) != mpack_ok || + flb_log_event_decoder_decode_timestamp(×tamp, &time) != 0) { + goto done; + } + + if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_START) { + in_group = FLB_TRUE; + } + else if (time.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { + in_group = FLB_FALSE; + } + else if (records < INT_MAX) { + records++; + } + + position = (bytes - offset) - mpack_reader_remaining(&reader, NULL); + + /* + * Never start the next window with an end marker: per window it would + * look like the end of a group opened by a previous append. + */ + if (!in_group && (position >= limit_bytes || records >= limit_records) && + (offset + position >= bytes || + !log_event_is_group_end((const char *) data + offset + position, + bytes - offset - position))) { + break; + } + } + + if (position > 0) { + *window_end = offset + position; + *window_records = records; + ret = 0; + } + +done: + /* flag early exits so unfinished compound types are not reported */ + if (ret != 0) { + mpack_reader_flag_error(&reader, mpack_error_invalid); + } + if (mpack_reader_destroy(&reader) != mpack_ok) { + ret = -1; + } + + return ret; +} + struct flb_mp_record_span { const char *base; size_t length; @@ -80,11 +556,28 @@ struct flb_mp_record_span { int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_size, char **out_buf, size_t *out_size) +{ + return flb_mp_normalize_log_buffer_groups_msgpack_ex(in_buf, in_size, out_buf, out_size, 0); +} + +/* + * Group normalization removes empty groups, end markers without a start and + * groups left open at the end of the buffer. 'flags' keeps the markers at the + * append boundaries when the caller knows the input already had them: a group + * split across appends keeps its first end marker before any start + * (FLB_MP_GROUPS_KEEP_LEADING_END) and its single open trailing group + * (FLB_MP_GROUPS_KEEP_TRAILING_OPEN). + */ +int flb_mp_normalize_log_buffer_groups_msgpack_ex(const void *in_buf, size_t in_size, + char **out_buf, size_t *out_size, + int flags) { int i; int ret; int entry_count; int stack_count; + int seen_start = FLB_FALSE; + int leading_end = FLB_FALSE; int *group_stack; int *group_has_content; struct flb_log_event log_event; @@ -172,12 +665,19 @@ int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_siz group_stack[stack_count] = i; group_has_content[stack_count] = FLB_FALSE; stack_count++; + seen_start = FLB_TRUE; continue; } if (entries[i].type == FLB_LOG_EVENT_GROUP_END) { if (stack_count == 0) { - entries[i].keep = FLB_FALSE; + /* the first end before any start closes a previous append's group */ + if ((flags & FLB_MP_GROUPS_KEEP_LEADING_END) && !seen_start && !leading_end) { + leading_end = FLB_TRUE; + } + else { + entries[i].keep = FLB_FALSE; + } continue; } @@ -199,6 +699,11 @@ int flb_mp_normalize_log_buffer_groups_msgpack(const void *in_buf, size_t in_siz } } + /* a single open group continues in the next append, nested ones are dropped */ + if ((flags & FLB_MP_GROUPS_KEEP_TRAILING_OPEN) && stack_count == 1) { + stack_count = 0; + } + while (stack_count > 0) { stack_count--; entries[group_stack[stack_count]].keep = FLB_FALSE; @@ -1030,210 +1535,288 @@ void flb_mp_accessor_destroy(struct flb_mp_accessor *mpa) } -static int mp_object_to_cfl(void **ptr, msgpack_object *o) +/* + * Convert a msgpack object into a CFL variant. When 'arena' is set, the + * variant and its children are allocated from it, otherwise from the heap. + */ +static struct cfl_variant *mp_variant_to_cfl(msgpack_object *object, struct cfl_arena *arena) { - int i; - int ret = -1; + struct cfl_variant *value; + struct cfl_variant *child; struct cfl_array *array; - struct cfl_kvlist *kvlist; - void *var; - msgpack_object key; - msgpack_object val; + struct cfl_kvlist *map; + msgpack_object_kv *pair; + size_t index; - switch(o->type) { + switch (object->type) { case MSGPACK_OBJECT_NIL: - var = cfl_variant_create_from_null(); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_null_in(arena); case MSGPACK_OBJECT_BOOLEAN: - var = cfl_variant_create_from_bool(o->via.boolean); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_bool_in(arena, object->via.boolean); case MSGPACK_OBJECT_POSITIVE_INTEGER: - var = cfl_variant_create_from_uint64(o->via.u64); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_uint64_in(arena, object->via.u64); case MSGPACK_OBJECT_NEGATIVE_INTEGER: - var = cfl_variant_create_from_int64(o->via.i64); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_int64_in(arena, object->via.i64); case MSGPACK_OBJECT_FLOAT32: case MSGPACK_OBJECT_FLOAT64: - var = cfl_variant_create_from_double(o->via.f64); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_double_in(arena, object->via.f64); case MSGPACK_OBJECT_STR: - var = cfl_variant_create_from_string_s((char *) o->via.str.ptr, - o->via.str.size, CFL_TRUE); - if (!var) { - return -1; - } - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; + return cfl_variant_create_from_string_s_in(arena, (char *) object->via.str.ptr, + object->via.str.size, CFL_TRUE); case MSGPACK_OBJECT_BIN: - var = cfl_variant_create_from_bytes((char *) o->via.str.ptr, - o->via.str.size, CFL_TRUE); - if (!var) { - return -1; - } - - *ptr = var; - ret = CFL_OBJECT_VARIANT; - break; - case MSGPACK_OBJECT_EXT: - /* we do not pack extension type content */ - *ptr = NULL; - ret = CFL_OBJECT_NONE; - break; + return cfl_variant_create_from_bytes_in(arena, (char *) object->via.bin.ptr, + object->via.bin.size, CFL_TRUE); case MSGPACK_OBJECT_ARRAY: - array = cfl_array_create(o->via.map.size); + array = cfl_array_create_in(arena, object->via.array.size); if (!array) { - return -1; + return NULL; } - ret = 0; - for (i = 0; i < o->via.map.size; i++) { - ret = mp_object_to_cfl((void *) &var, &o->via.array.ptr[i]); - if (ret == CFL_OBJECT_KVLIST) { - ret = cfl_array_append_kvlist(array, var); - } - else if (ret == CFL_OBJECT_VARIANT) { - ret = cfl_array_append(array, var); - } - else if (ret == CFL_OBJECT_ARRAY) { - ret = cfl_array_append_array(array, var); - } - else { - ret = -1; - break; + for (index = 0; index < object->via.array.size; index++) { + child = mp_variant_to_cfl(&object->via.array.ptr[index], arena); + if (!child || cfl_array_append(array, child) != 0) { + cfl_variant_destroy(child); + cfl_array_destroy(array); + return NULL; } } - if (ret == -1) { + value = cfl_variant_create_from_array_in(arena, array); + if (!value) { cfl_array_destroy(array); - return -1; } - - *ptr = array; - ret = CFL_OBJECT_ARRAY; - break; + return value; case MSGPACK_OBJECT_MAP: - kvlist = cfl_kvlist_create(); - if (!kvlist) { - return -1; + map = cfl_kvlist_create_in(arena); + if (!map) { + return NULL; } - ret = 0; - for (i = 0; i < o->via.map.size; i++) { - key = o->via.map.ptr[i].key; - val = o->via.map.ptr[i].val; - - /* force key type to be strin, otherwise just abort */ - if (key.type != MSGPACK_OBJECT_STR) { - ret = -1; - break; - } + for (index = 0; index < object->via.map.size; index++) { + pair = &object->via.map.ptr[index]; - /* key variant is ready, now we need the value variant */ - ret = mp_object_to_cfl((void *) &var, &val); - if (ret == -1) { - break; + /* only string keys are supported */ + if (pair->key.type != MSGPACK_OBJECT_STR) { + cfl_kvlist_destroy(map); + return NULL; } - if (ret == CFL_OBJECT_KVLIST) { - ret = cfl_kvlist_insert_kvlist_s(kvlist, - (char *) key.via.str.ptr, key.via.str.size, - var); - } - else if (ret == CFL_OBJECT_VARIANT) { - ret = cfl_kvlist_insert_s(kvlist, - (char *) key.via.str.ptr, key.via.str.size, - var); - } - else if (ret == CFL_OBJECT_ARRAY) { - ret = cfl_kvlist_insert_array_s(kvlist, - (char *) key.via.str.ptr, key.via.str.size, - var); - } - else { - ret = -1; - break; + child = mp_variant_to_cfl(&pair->val, arena); + if (!child || cfl_kvlist_insert_s(map, (char *) pair->key.via.str.ptr, + pair->key.via.str.size, child) != 0) { + cfl_variant_destroy(child); + cfl_kvlist_destroy(map); + return NULL; } } - if (ret == -1) { - cfl_kvlist_destroy(kvlist); - return -1; + value = cfl_variant_create_from_kvlist_in(arena, map); + if (!value) { + cfl_kvlist_destroy(map); } - - *ptr = kvlist; - ret = CFL_OBJECT_KVLIST; - break; + return value; default: - break; + /* extension types are not converted */ + return NULL; } - - return ret; } +/* Only maps (kvlist) and arrays can be converted to a cfl_object */ +static struct cfl_object *mp_object_to_cfl_in(msgpack_object *object, struct cfl_arena *arena) +{ + struct cfl_object *result; + + if (object->type != MSGPACK_OBJECT_MAP && object->type != MSGPACK_OBJECT_ARRAY) { + return NULL; + } + + result = cfl_object_create(); + if (!result) { + return NULL; + } + + result->variant = mp_variant_to_cfl(object, arena); + if (!result->variant) { + cfl_object_destroy(result); + return NULL; + } + + if (object->type == MSGPACK_OBJECT_MAP) { + result->type = CFL_OBJECT_KVLIST; + } + else { + result->type = CFL_OBJECT_ARRAY; + } + + return result; +} /* Convert a msgpack object to a cfl_object */ struct cfl_object *flb_mp_object_to_cfl(msgpack_object *o) { - int ret; - void *out = NULL; - struct cfl_object *obj; + return mp_object_to_cfl_in(o, NULL); +} - /* For now, only allow to convert to map (kvlist) or array */ - if (o->type != MSGPACK_OBJECT_MAP && o->type != MSGPACK_OBJECT_ARRAY) { +/* + * Bytes a chunk value pool can hold before new record trees use the heap, and + * its arena block size. Trees already in the pool can keep growing, so edits + * on them never fail because the pool is full. + */ +#define FLB_MP_VALUE_POOL_LIMIT (1024 * 1024) +#define FLB_MP_VALUE_POOL_BLOCK 16384 + +/* Prefix of every pool allocation, it keeps the size for accounting */ +union value_pool_header { + size_t size; + long double alignment; + void *pointer; +}; + +static void *value_pool_malloc(void *context, size_t size) +{ + struct flb_mp_chunk_cobj *chunk = context; + union value_pool_header *header; + + if (size > SIZE_MAX - sizeof(*header)) { return NULL; } - obj = cfl_object_create(); - if (!obj) { + header = flb_malloc(sizeof(*header) + size); + if (!header) { + flb_errno(); + return NULL; + } + + header->size = sizeof(*header) + size; + chunk->value_pool_bytes += header->size; + + return header + 1; +} + +static void value_pool_free(void *context, void *pointer) +{ + struct flb_mp_chunk_cobj *chunk = context; + union value_pool_header *header = (union value_pool_header *) pointer - 1; + + chunk->value_pool_bytes -= header->size; + flb_free(header); +} + +void flb_mp_chunk_cobj_value_pool_enable(struct flb_mp_chunk_cobj *chunk) +{ + struct cfl_arena_options options; + + if (chunk->value_pool) { + return; + } + + /* fixed size blocks, accounted against the chunk through the hooks */ + cfl_arena_options_init(&options); + options.chunk_size = FLB_MP_VALUE_POOL_BLOCK; + options.maximum_chunk_size = FLB_MP_VALUE_POOL_BLOCK; + options.large_object_threshold = FLB_MP_VALUE_POOL_BLOCK; + options.malloc_fn = value_pool_malloc; + options.free_fn = value_pool_free; + options.allocator_context = chunk; + + chunk->value_pool = cfl_arena_create_with_options(&options); + if (chunk->value_pool) { + cfl_arena_external_cache_limit_set(chunk->value_pool, 0); + } +} + +struct cfl_arena *flb_mp_chunk_cobj_value_pool_get(struct flb_mp_chunk_cobj *chunk, + size_t estimated_bytes) +{ + size_t available; + + if (!chunk || !chunk->value_pool || + chunk->value_pool_bytes >= FLB_MP_VALUE_POOL_LIMIT) { return NULL; } - ret = mp_object_to_cfl(&out, o); - if (ret < 0) { - cfl_object_destroy(obj); + available = FLB_MP_VALUE_POOL_LIMIT - chunk->value_pool_bytes; + + /* + * Leave room for a new arena block and its allocation headers. Oversized + * roots use heap ownership as a whole, never mixed container ownership. + */ + if (available < 2 * FLB_MP_VALUE_POOL_BLOCK || + estimated_bytes > available - 2 * FLB_MP_VALUE_POOL_BLOCK) { return NULL; } - ret = cfl_object_set(obj, ret, out); - if (ret == -1) { - if (ret == CFL_OBJECT_KVLIST) { - cfl_kvlist_destroy(out); + return chunk->value_pool; +} + +/* + * Upper estimate for arena nodes, aligned slots, kvpairs and copied map keys. + * MessagePack string/byte payloads themselves remain borrowed from the decoder. + */ +static size_t mp_value_pool_cost(msgpack_object *object, int depth) +{ + size_t cost = 512; + size_t child; + size_t index; + msgpack_object *value; + + if (depth >= 64) { + return SIZE_MAX; + } + + if (object->type != MSGPACK_OBJECT_MAP && object->type != MSGPACK_OBJECT_ARRAY) { + return cost; + } + + /* maps and arrays share the same 'size' layout in the msgpack union */ + for (index = 0; index < object->via.array.size; index++) { + if (object->type == MSGPACK_OBJECT_MAP) { + value = &object->via.map.ptr[index].val; + + child = 0; + if (object->via.map.ptr[index].key.type == MSGPACK_OBJECT_STR) { + child = object->via.map.ptr[index].key.via.str.size; + } + + if (cost > SIZE_MAX - 512 || child > (SIZE_MAX - cost - 512) / 2) { + return SIZE_MAX; + } + cost += 512 + 2 * child; + } + else { + value = &object->via.array.ptr[index]; } - else if (ret == CFL_OBJECT_ARRAY) { - cfl_array_destroy(out); + + child = mp_value_pool_cost(value, depth + 1); + if (child > SIZE_MAX - cost) { + return SIZE_MAX; } - cfl_object_destroy(obj); - return NULL; + cost += child; } - return obj; + return cost; +} + +/* + * Convert a decoded msgpack object for a chunk record. The chunk value pool is + * used when enabled and large enough, falling back to heap allocations. + */ +static struct cfl_object *chunk_object_from_msgpack(struct flb_mp_chunk_cobj *chunk, + msgpack_object *object) +{ + struct cfl_arena *arena = NULL; + struct cfl_object *result; + + if (chunk->value_pool) { + arena = flb_mp_chunk_cobj_value_pool_get(chunk, mp_value_pool_cost(object, 0)); + } + + result = mp_object_to_cfl_in(object, arena); + if (!result && arena) { + result = mp_object_to_cfl_in(object, NULL); + } + + return result; } static int mp_cfl_to_msgpack(struct cfl_variant *var, @@ -1347,20 +1930,76 @@ int flb_mp_cfl_to_msgpack(struct cfl_object *obj, char **out_buf, size_t *out_si return 0; } +/* Records per pool block and maximum number of blocks per chunk */ +#define FLB_MP_RECORD_BLOCK_ENTRIES 64 +#define FLB_MP_RECORD_BLOCK_LIMIT 8 + +/* + * A block has one owner reference plus one reference per live record. Moving + * records between chunks does not change their storage lifetime. + */ +struct flb_mp_record_block { + struct flb_mp_record_block *next; + size_t references; + size_t used; + struct flb_mp_chunk_record records[FLB_MP_RECORD_BLOCK_ENTRIES]; +}; + +static void record_block_release(struct flb_mp_record_block *block) +{ + block->references--; + if (block->references == 0) { + flb_free(block); + } +} + +static void record_storage_release(struct flb_mp_chunk_record *record) +{ + if (record->storage_block) { + record_block_release(record->storage_block); + } + else { + flb_free(record); + } +} + struct flb_mp_chunk_record *flb_mp_chunk_record_create(struct flb_mp_chunk_cobj *chunk_cobj) { struct flb_mp_chunk_record *record; + struct flb_mp_record_block *block; + + if (chunk_cobj && chunk_cobj->record_pool_enabled) { + block = chunk_cobj->record_pool; + /* the current block is full (or missing), try to add a new one */ + if ((!block || block->used == FLB_MP_RECORD_BLOCK_ENTRIES) && + chunk_cobj->record_pool_blocks < FLB_MP_RECORD_BLOCK_LIMIT) { + block = flb_calloc(1, sizeof(struct flb_mp_record_block)); + if (!block) { + flb_errno(); + } + else { + block->references = 1; + block->next = chunk_cobj->record_pool; + chunk_cobj->record_pool = block; + chunk_cobj->record_pool_blocks++; + } + } + + if (block && block->used < FLB_MP_RECORD_BLOCK_ENTRIES) { + record = &block->records[block->used++]; + record->storage_block = block; + block->references++; + return record; + } + } + + /* pool disabled or exhausted: allocate the record individually */ record = flb_calloc(1, sizeof(struct flb_mp_chunk_record)); if (!record) { flb_errno(); return NULL; } - record->modified = FLB_FALSE; - record->cobj_group_metadata = NULL; - record->cobj_group_attributes = NULL; - record->owns_group_metadata = FLB_FALSE; - record->owns_group_attributes = FLB_FALSE; return record; } @@ -1389,118 +2028,112 @@ struct flb_mp_chunk_cobj *flb_mp_chunk_cobj_create(struct flb_log_event_encoder return chunk_cobj; } -static int generate_empty_msgpack_map(char **out_buf, size_t *out_size) +/* + * Pack a record field into the scratch buffer, an empty map when the field is + * not set. The buffer capacity is reused by every field of an encode call. + */ +static int encode_cfl_field(struct cfl_object *obj, msgpack_sbuffer *buffer, + msgpack_packer *packer) { - msgpack_sbuffer mp_sbuf; - msgpack_packer mp_pck; + msgpack_sbuffer_clear(buffer); - /* initialize msgpack buffer */ - msgpack_sbuffer_init(&mp_sbuf); - msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); - - msgpack_pack_map(&mp_pck, 0); + if (!obj) { + return msgpack_pack_map(packer, 0); + } - *out_buf = mp_sbuf.data; - *out_size = mp_sbuf.size; + if (obj->type == CFL_OBJECT_NONE || !obj->variant) { + return -1; + } - return 0; + return mp_cfl_to_msgpack(obj->variant, buffer, packer); } -int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size) +/* + * Append the chunk records to its log encoder output. A failure resets the + * encoder, including the output of previous appends. + */ +int flb_mp_chunk_cobj_encode_append(struct flb_mp_chunk_cobj *chunk_cobj) { int ret; - int record_type; - char *mp_buf; - size_t mp_size; struct cfl_list *head; struct flb_mp_chunk_record *record; + struct cfl_object *body; + msgpack_sbuffer buffer; + msgpack_packer packer; if (!chunk_cobj) { return -1; } + msgpack_sbuffer_init(&buffer); + msgpack_packer_init(&packer, &buffer, msgpack_sbuffer_write); + /* Iterate all records */ cfl_list_foreach(head, &chunk_cobj->records) { record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); ret = flb_log_event_encoder_begin_record(chunk_cobj->log_encoder); - if (ret == -1) { - return -1; + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto failure; } - ret = flb_log_event_encoder_set_timestamp(chunk_cobj->log_encoder, &record->event.timestamp); - if (ret == -1) { - return -1; + ret = flb_log_event_encoder_set_timestamp(chunk_cobj->log_encoder, + &record->event.timestamp); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto failure; } - /* Determine record type from timestamp */ - if (record->event.timestamp.tm.tv_sec >= 0) { - record_type = FLB_LOG_EVENT_NORMAL; - } - else if (record->event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_START) { - record_type = FLB_LOG_EVENT_GROUP_START; - } - else if (record->event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_END) { - record_type = FLB_LOG_EVENT_GROUP_END; - } - else { - record_type = FLB_LOG_EVENT_NORMAL; + /* metadata */ + if (encode_cfl_field(record->cobj_metadata, &buffer, &packer) != 0) { + goto failure; } - - if (record->cobj_metadata) { - ret = flb_mp_cfl_to_msgpack(record->cobj_metadata, &mp_buf, &mp_size); - if (ret == -1) { - return -1; - } - } - else { - ret = generate_empty_msgpack_map(&mp_buf, &mp_size); - if (ret == -1) { - return -1; - } - } - - ret = flb_log_event_encoder_set_metadata_from_raw_msgpack(chunk_cobj->log_encoder, mp_buf, mp_size); + ret = flb_log_event_encoder_set_metadata_from_raw_msgpack(chunk_cobj->log_encoder, + buffer.data, buffer.size); if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_free(mp_buf); - return -1; + goto failure; } - flb_free(mp_buf); - /* For group start records, use group attributes as body if available */ - if (record_type == FLB_LOG_EVENT_GROUP_START && record->cobj_group_attributes) { - ret = flb_mp_cfl_to_msgpack(record->cobj_group_attributes, &mp_buf, &mp_size); - if (ret == -1) { - return -1; - } - } - else if (record->cobj_record) { - ret = flb_mp_cfl_to_msgpack(record->cobj_record, &mp_buf, &mp_size); - if (ret == -1) { - return -1; - } + /* for group start records, use group attributes as body if available */ + body = record->cobj_record; + if (record->event.timestamp.tm.tv_sec == FLB_LOG_EVENT_GROUP_START && + record->cobj_group_attributes) { + body = record->cobj_group_attributes; } - else { - ret = generate_empty_msgpack_map(&mp_buf, &mp_size); - if (ret == -1) { - return -1; - } + + if (encode_cfl_field(body, &buffer, &packer) != 0) { + goto failure; } - ret = flb_log_event_encoder_set_body_from_raw_msgpack(chunk_cobj->log_encoder, mp_buf, mp_size); + ret = flb_log_event_encoder_set_body_from_raw_msgpack(chunk_cobj->log_encoder, + buffer.data, buffer.size); if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_free(mp_buf); - return -1; + goto failure; } - flb_free(mp_buf); ret = flb_log_event_encoder_commit_record(chunk_cobj->log_encoder); - if (ret == -1) { - return -1; + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto failure; } } + msgpack_sbuffer_destroy(&buffer); + return 0; + +failure: + /* never leave partial records behind for the next chunk using this encoder */ + flb_log_event_encoder_reset(chunk_cobj->log_encoder); + msgpack_sbuffer_destroy(&buffer); + + return -1; +} + +int flb_mp_chunk_cobj_encode(struct flb_mp_chunk_cobj *chunk_cobj, char **out_buf, size_t *out_size) +{ + if (flb_mp_chunk_cobj_encode_append(chunk_cobj) != 0) { + return -1; + } + /* set new output buffer */ *out_buf = chunk_cobj->log_encoder->output_buffer; *out_size = chunk_cobj->log_encoder->output_length; @@ -1567,9 +2200,17 @@ static int cobj_remove_list_add(struct flb_mp_chunk_record **remove_list, } int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj) +{ + return flb_mp_chunk_cobj_normalize_groups_ex(chunk_cobj, 0); +} + +/* see flb_mp_normalize_log_buffer_groups_msgpack_ex() for 'flags' */ +int flb_mp_chunk_cobj_normalize_groups_ex(struct flb_mp_chunk_cobj *chunk_cobj, int flags) { int ret; int record_type; + int seen_start = FLB_FALSE; + int leading_end = FLB_FALSE; size_t index; size_t stack_size; size_t stack_count; @@ -1627,12 +2268,19 @@ int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj) group_has_content[stack_count] = FLB_FALSE; stack_count++; } + seen_start = FLB_TRUE; continue; } if (record_type == FLB_LOG_EVENT_GROUP_END) { if (stack_count == 0) { - cobj_remove_list_add(remove_list, stack_size, &remove_count, record); + /* the first end before any start closes a previous append's group */ + if ((flags & FLB_MP_GROUPS_KEEP_LEADING_END) && !seen_start && !leading_end) { + leading_end = FLB_TRUE; + } + else { + cobj_remove_list_add(remove_list, stack_size, &remove_count, record); + } continue; } @@ -1655,6 +2303,11 @@ int flb_mp_chunk_cobj_normalize_groups(struct flb_mp_chunk_cobj *chunk_cobj) } } + /* a single open group continues in the next append, nested ones are dropped */ + if ((flags & FLB_MP_GROUPS_KEEP_TRAILING_OPEN) && stack_count == 1) { + stack_count = 0; + } + while (stack_count > 0) { stack_count--; cobj_remove_list_add(remove_list, stack_size, &remove_count, @@ -1677,6 +2330,7 @@ int flb_mp_chunk_cobj_destroy(struct flb_mp_chunk_cobj *chunk_cobj) struct cfl_list *tmp; struct cfl_list *head; struct flb_mp_chunk_record *record; + struct flb_mp_record_block *block; if (!chunk_cobj) { return -1; @@ -1699,9 +2353,17 @@ int flb_mp_chunk_cobj_destroy(struct flb_mp_chunk_cobj *chunk_cobj) cfl_object_destroy(record->cobj_group_attributes); } cfl_list_del(&record->_head); - flb_free(record); + record_storage_release(record); + } + + /* drop the owner reference, blocks still used by moved records survive */ + while (chunk_cobj->record_pool) { + block = chunk_cobj->record_pool; + chunk_cobj->record_pool = block->next; + record_block_release(block); } + cfl_arena_destroy(chunk_cobj->value_pool); flb_free(chunk_cobj); return 0; } @@ -1747,20 +2409,21 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, ret = flb_log_event_decoder_next(chunk_cobj->log_decoder, &record->event); if (ret != FLB_EVENT_DECODER_SUCCESS) { - flb_free(record); + record_storage_release(record); return -1; } - record->cobj_metadata = flb_mp_object_to_cfl(record->event.metadata); + record->cobj_metadata = chunk_object_from_msgpack(chunk_cobj, + record->event.metadata); if (!record->cobj_metadata) { - flb_free(record); + record_storage_release(record); return FLB_MP_CHUNK_RECORD_ERROR; } - record->cobj_record = flb_mp_object_to_cfl(record->event.body); + record->cobj_record = chunk_object_from_msgpack(chunk_cobj, record->event.body); if (!record->cobj_record) { cfl_object_destroy(record->cobj_metadata); - flb_free(record); + record_storage_release(record); return -1; } @@ -1768,7 +2431,7 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, if (ret != FLB_EVENT_DECODER_SUCCESS) { cfl_object_destroy(record->cobj_record); cfl_object_destroy(record->cobj_metadata); - flb_free(record); + record_storage_release(record); return FLB_MP_CHUNK_RECORD_ERROR; } @@ -1804,14 +2467,15 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, record->event.group_metadata && (record->event.group_metadata->type == MSGPACK_OBJECT_MAP || record->event.group_metadata->type == MSGPACK_OBJECT_ARRAY)) { - record->cobj_group_metadata = flb_mp_object_to_cfl(record->event.group_metadata); + record->cobj_group_metadata = + chunk_object_from_msgpack(chunk_cobj, record->event.group_metadata); if (!record->cobj_group_metadata) { if (record->owns_group_attributes && record->cobj_group_attributes) { cfl_object_destroy(record->cobj_group_attributes); } cfl_object_destroy(record->cobj_record); cfl_object_destroy(record->cobj_metadata); - flb_free(record); + record_storage_release(record); return FLB_MP_CHUNK_RECORD_ERROR; } record->owns_group_metadata = FLB_TRUE; @@ -1824,14 +2488,15 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, record->event.group_attributes && (record->event.group_attributes->type == MSGPACK_OBJECT_MAP || record->event.group_attributes->type == MSGPACK_OBJECT_ARRAY)) { - record->cobj_group_attributes = flb_mp_object_to_cfl(record->event.group_attributes); + record->cobj_group_attributes = + chunk_object_from_msgpack(chunk_cobj, record->event.group_attributes); if (!record->cobj_group_attributes) { if (record->owns_group_metadata && record->cobj_group_metadata) { cfl_object_destroy(record->cobj_group_metadata); } cfl_object_destroy(record->cobj_record); cfl_object_destroy(record->cobj_metadata); - flb_free(record); + record_storage_release(record); return FLB_MP_CHUNK_RECORD_ERROR; } record->owns_group_attributes = FLB_TRUE; @@ -1967,7 +2632,7 @@ int flb_mp_chunk_cobj_record_destroy(struct flb_mp_chunk_cobj *chunk_cobj, } cfl_list_del(&record->_head); - flb_free(record); + record_storage_release(record); return 0; } diff --git a/src/flb_processor.c b/src/flb_processor.c index 7f04146ff6f..836fe42f66e 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -1314,12 +1314,365 @@ int flb_processor_is_active(struct flb_processor *proc) * context for metrics, a 'CTraces' context for traces or a 'CProfiles' context for * profiles. */ +/* + * Decoder and encoder owned by one processor run. The decoded chunk lives + * across several units, so it must not borrow the first unit's decoder: its + * lock is released before the next units run and another worker thread + * could reset it while records are still being decoded. + */ +struct chunk_codec { + int ready; + struct flb_log_event_decoder decoder; + struct flb_log_event_encoder encoder; +}; + +static int chunk_codec_prepare(struct chunk_codec *codec) +{ + if (codec->ready) { + return 0; + } + + if (flb_log_event_decoder_init(&codec->decoder, NULL, 0) != FLB_EVENT_DECODER_SUCCESS) { + return -1; + } + /* chunks keep group markers as records, like processor instance decoders */ + flb_log_event_decoder_read_groups(&codec->decoder, FLB_TRUE); + + if (flb_log_event_encoder_init(&codec->encoder, + FLB_LOG_EVENT_FORMAT_DEFAULT) != FLB_EVENT_ENCODER_SUCCESS) { + flb_log_event_decoder_destroy(&codec->decoder); + return -1; + } + + codec->ready = FLB_TRUE; + return 0; +} + +static void chunk_codec_destroy(struct chunk_codec *codec) +{ + if (!codec->ready) { + return; + } + + flb_log_event_decoder_destroy(&codec->decoder); + flb_log_event_encoder_destroy(&codec->encoder); + codec->ready = FLB_FALSE; +} + +/* + * Large log buffers, such as complete chunks reaching output processors, are + * processed in windows when every native unit of the run is record local: + * each window is decoded, processed by all the units, encoded and released + * before the next one, so decoded records never hold more than one window. + * Windows only close outside groups, keeping every group in one window. + */ +#define FLB_PROCESSOR_LOG_WINDOW_BYTES 65536 +#define FLB_PROCESSOR_LOG_WINDOW_RECORDS 1024 + +#define LOG_WINDOWS_DONE 0 +#define LOG_WINDOWS_SKIP 1 +#define LOG_WINDOWS_ERROR -1 + +struct log_window_unit { + struct flb_processor_unit *pu; + int in_items; + int out_items; + int error; +}; + +/* last unit of the native run starting at 'head' if windows can be used */ +static struct mk_list *log_window_run_end(struct mk_list *head, struct mk_list *list, + int *units) +{ + struct mk_list *last = NULL; + struct flb_processor_unit *pu; + struct flb_processor_instance *p_ins; + + *units = 0; + for (; head != list; head = head->next) { + pu = mk_list_entry(head, struct flb_processor_unit, _head); + if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { + break; + } + + p_ins = pu->ctx; + if (p_ins->p->cb_process_logs == NULL || + (p_ins->p->flags & FLB_PROCESSOR_FLAG_LOG_WINDOWS) == 0) { + return NULL; + } + + last = head; + (*units)++; + } + + return last; +} + +struct log_window { + size_t end; + int records; +}; + +/* window boundaries for the complete buffer, NULL if one window is enough */ +static struct log_window *log_windows_scan(void *data, size_t data_size, int *count) +{ + int records; + int capacity = 0; + size_t offset = 0; + size_t end; + struct log_window *tmp; + struct log_window *windows = NULL; + + *count = 0; + while (offset < data_size) { + if (flb_mp_log_window_next(data, data_size, offset, + FLB_PROCESSOR_LOG_WINDOW_BYTES, + FLB_PROCESSOR_LOG_WINDOW_RECORDS, + &end, &records) != 0) { + /* the single chunk path reports invalid content as usual */ + flb_free(windows); + return NULL; + } + + if (*count == capacity) { + capacity = capacity ? capacity * 2 : 32; + tmp = flb_realloc(windows, sizeof(struct log_window) * capacity); + if (!tmp) { + flb_errno(); + flb_free(windows); + return NULL; + } + windows = tmp; + } + + windows[*count].end = end; + windows[*count].records = records; + (*count)++; + offset = end; + } + + if (*count < 2) { + flb_free(windows); + return NULL; + } + + return windows; +} + +/* TRUE when a decoded chunk holds group markers */ +static int chunk_has_group_markers(struct flb_mp_chunk_cobj *chunk) +{ + int type; + struct cfl_list *head; + struct flb_mp_chunk_record *record; + + cfl_list_foreach(head, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + if (flb_log_event_decoder_get_record_type(&record->event, &type) == 0 && + (type == FLB_LOG_EVENT_GROUP_START || type == FLB_LOG_EVENT_GROUP_END)) { + return FLB_TRUE; + } + } + + return FLB_FALSE; +} + +/* + * Group boundary flags of the processor input, scanned once and only when a + * chunk to normalize has group markers: plain logs never pay for the scan. + */ +static int input_group_flags(int *cache, void *data, size_t data_size) +{ + if (*cache < 0) { + *cache = flb_mp_log_group_boundaries(data, data_size); + } + return *cache; +} + +static int run_log_windows(struct flb_processor *proc, struct chunk_codec *codec, + struct mk_list *first, int unit_count, + const char *tag, size_t tag_len, + void *data, size_t data_size, + void **out_buf, size_t *out_size, + int *out_records, int *out_normalized, + void *input, size_t input_size, int *group_flags, + const char *metrics_scope, + const char *metrics_owner, + const char *metrics_signal) +{ + int i; + int w; + int ret; + int result = LOG_WINDOWS_ERROR; + int records; + int total_records = 0; + int normalized = FLB_TRUE; + int window_count; + int window_flags; + int input_flags; + int seen_start = FLB_FALSE; + size_t offset; + struct mk_list *head; + struct log_window *windows; + struct log_window_unit *units; + struct flb_processor_instance *p_ins; + struct flb_mp_chunk_cobj *chunk_cobj = NULL; + + windows = log_windows_scan(data, data_size, &window_count); + if (!windows) { + return LOG_WINDOWS_SKIP; + } + + units = flb_calloc(unit_count, sizeof(struct log_window_unit)); + if (!units) { + flb_errno(); + flb_free(windows); + return LOG_WINDOWS_SKIP; + } + + if (chunk_codec_prepare(codec) != 0) { + flb_free(units); + flb_free(windows); + return LOG_WINDOWS_SKIP; + } + + head = first; + for (i = 0; i < unit_count; i++) { + units[i].pu = mk_list_entry(head, struct flb_processor_unit, _head); + head = head->next; + } + + offset = 0; + for (w = 0; w < window_count; w++) { + flb_log_event_decoder_reset(&codec->decoder, (char *) data + offset, + windows[w].end - offset); + chunk_cobj = flb_mp_chunk_cobj_create(&codec->encoder, &codec->decoder); + if (!chunk_cobj) { + goto error; + } + + records = windows[w].records; + for (i = 0; i < unit_count; i++) { + p_ins = units[i].pu->ctx; + + ret = acquire_lock(&units[i].pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + if (ret != FLB_TRUE) { + units[i].error = FLB_TRUE; + goto error; + } + + chunk_cobj->condition = units[i].pu->condition; + ret = p_ins->p->cb_process_logs(p_ins, chunk_cobj, tag, tag_len); + if (ret != FLB_PROCESSOR_SUCCESS) { + units[i].error = FLB_TRUE; + flb_warn("[processor] failed to process chunk"); + } + chunk_cobj->record_pos = NULL; + chunk_cobj->condition = NULL; + + release_lock(&units[i].pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + + units[i].in_items += records; + records = flb_mp_chunk_cobj_count_log_records(chunk_cobj); + units[i].out_items += records; + } + + /* + * Boundary markers of the input stay with the window holding them: a + * leading end until a group starts, the open group in the last window. + */ + window_flags = 0; + if (chunk_has_group_markers(chunk_cobj)) { + input_flags = input_group_flags(group_flags, input, input_size); + if (!seen_start) { + window_flags |= input_flags & FLB_MP_GROUPS_KEEP_LEADING_END; + } + if (w == window_count - 1) { + window_flags |= input_flags & FLB_MP_GROUPS_KEEP_TRAILING_OPEN; + } + } + /* only inputs opening with a continued group need the start count */ + if (*group_flags > 0 && (*group_flags & FLB_MP_GROUPS_KEEP_LEADING_END) && + !seen_start && + flb_mp_log_group_starts((char *) data + offset, windows[w].end - offset) > 0) { + seen_start = FLB_TRUE; + } + + if (flb_mp_chunk_cobj_normalize_groups_ex(chunk_cobj, window_flags) != 0) { + normalized = FLB_FALSE; + } + total_records += flb_mp_chunk_cobj_count_log_records(chunk_cobj); + + if (cfl_list_size(&chunk_cobj->records) > 0 && + flb_mp_chunk_cobj_encode_append(chunk_cobj) != 0) { + goto error; + } + + flb_mp_chunk_cobj_destroy(chunk_cobj); + chunk_cobj = NULL; + offset = windows[w].end; + } + + if (total_records == 0 || codec->encoder.output_length == 0) { + flb_log_event_encoder_reset(&codec->encoder); + *out_buf = NULL; + *out_size = 0; + total_records = 0; + } + else { + *out_buf = codec->encoder.output_buffer; + *out_size = codec->encoder.output_length; + flb_log_event_encoder_claim_internal_buffer_ownership(&codec->encoder); + } + + *out_records = total_records; + *out_normalized = normalized; + result = LOG_WINDOWS_DONE; + goto exit; + +error: + if (chunk_cobj) { + flb_mp_chunk_cobj_destroy(chunk_cobj); + } + flb_log_event_encoder_reset(&codec->encoder); + +exit: + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); + + for (i = 0; i < unit_count; i++) { + processor_metrics_update(proc, units[i].pu, + metrics_scope, metrics_owner, metrics_signal, + units[i].in_items, units[i].out_items, + FLB_TRUE, units[i].error); + } + flb_free(units); + flb_free(windows); + + return result; +} + int flb_processor_run(struct flb_processor *proc, size_t starting_stage, int type, const char *tag, size_t tag_len, void *data, size_t data_size, void **out_buf, size_t *out_size) +{ + return flb_processor_run_with_count(proc, starting_stage, type, tag, tag_len, + data, data_size, out_buf, out_size, -1, NULL); +} + +static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, + size_t starting_stage, + int type, + const char *tag, size_t tag_len, + void *data, size_t data_size, + void **out_buf, size_t *out_size, + int input_records, int *output_records) { int ret; int pu_error; @@ -1327,6 +1680,8 @@ int flb_processor_run(struct flb_processor *proc, int unit_out_items; int item_metrics_available; int finalize; + int log_buffer_normalized = FLB_FALSE; + int log_buffer_records = input_records; void *cur_buf = NULL; size_t cur_size; void *tmp_buf = NULL; @@ -1338,6 +1693,11 @@ int flb_processor_run(struct flb_processor *proc, struct flb_filter_instance *f_ins; struct flb_processor_instance *p_ins; struct flb_mp_chunk_cobj *chunk_cobj = NULL; + struct mk_list *run_end; + int run_units; + int window_records; + int window_normalized; + int group_flags = -1; const char *metrics_scope; const char *metrics_owner; const char *metrics_signal; @@ -1349,6 +1709,10 @@ int flb_processor_run(struct flb_processor *proc, char *name; #endif + if (output_records) { + *output_records = -1; + } + if (type == FLB_PROCESSOR_LOGS) { list = &proc->logs; } @@ -1395,11 +1759,59 @@ int flb_processor_run(struct flb_processor *proc, item_metrics_available = FLB_FALSE; if (type == FLB_PROCESSOR_LOGS) { - unit_in_items = flb_mp_count_log_records(cur_buf, cur_size); + if (!chunk_cobj && log_buffer_records >= 0) { + unit_in_items = log_buffer_records; + } + else { + unit_in_items = flb_mp_count_log_records(cur_buf, cur_size); + } unit_out_items = unit_in_items; item_metrics_available = FLB_TRUE; } + /* large buffers run record local native units in bounded windows */ + if (type == FLB_PROCESSOR_LOGS && chunk_cobj == NULL && + pu->unit_type == FLB_PROCESSOR_UNIT_NATIVE && + cur_size > FLB_PROCESSOR_LOG_WINDOW_BYTES) { + run_end = log_window_run_end(head, list, &run_units); + if (run_end != NULL) { + ret = run_log_windows(proc, codec, head, run_units, tag, tag_len, + cur_buf, cur_size, &tmp_buf, &tmp_size, + &window_records, &window_normalized, + data, data_size, &group_flags, + metrics_scope, metrics_owner, metrics_signal); + if (ret == LOG_WINDOWS_ERROR) { + if (cur_buf != data) { + flb_free(cur_buf); + } + return -1; + } + else if (ret == LOG_WINDOWS_DONE) { + if (cur_buf != data) { + flb_free(cur_buf); + } + + if (tmp_buf == NULL) { + *out_buf = NULL; + *out_size = 0; + if (output_records) { + *output_records = 0; + } + return 0; + } + + cur_buf = tmp_buf; + cur_size = tmp_size; + log_buffer_records = window_records; + log_buffer_normalized = window_normalized; + + /* continue after the last unit of the run */ + head = run_end; + continue; + } + } + } + ret = acquire_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); @@ -1436,7 +1848,12 @@ int flb_processor_run(struct flb_processor *proc, } #ifdef FLB_HAVE_METRICS name = (char *) (flb_filter_name(f_ins)); - in_records = flb_mp_count_log_records(cur_buf, cur_size); + if (log_buffer_records >= 0) { + in_records = log_buffer_records; + } + else { + in_records = flb_mp_count_log_records(cur_buf, cur_size); + } cmt_counter_add(f_ins->cmt_records, ts, in_records, 1, (char *[]) {name}); cmt_counter_add(f_ins->cmt_bytes, ts, tmp_size, @@ -1500,6 +1917,8 @@ int flb_processor_run(struct flb_processor *proc, cur_buf = tmp_buf; cur_size = tmp_size; out_records = flb_mp_count_log_records(tmp_buf, tmp_size); + log_buffer_normalized = FLB_FALSE; + log_buffer_records = out_records; unit_out_items = out_records; #ifdef FLB_HAVE_METRICS if (out_records > in_records) { @@ -1546,12 +1965,30 @@ int flb_processor_run(struct flb_processor *proc, * until no more processors exists or the next one is a "filter" type processor. */ if (!chunk_cobj) { - flb_log_event_decoder_reset(p_ins->log_decoder, cur_buf, cur_size); + if (chunk_codec_prepare(codec) != 0) { + if (cur_buf != data) { + flb_free(cur_buf); + } + pu_error = FLB_TRUE; + processor_metrics_update(proc, pu, + metrics_scope, + metrics_owner, + metrics_signal, + unit_in_items, + unit_out_items, + item_metrics_available, + pu_error); + release_lock(&pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + return -1; + } + flb_log_event_decoder_reset(&codec->decoder, cur_buf, cur_size); /* create the context */ - chunk_cobj = flb_mp_chunk_cobj_create(p_ins->log_encoder, p_ins->log_decoder); + chunk_cobj = flb_mp_chunk_cobj_create(&codec->encoder, &codec->decoder); if (chunk_cobj == NULL) { - flb_log_event_decoder_reset(p_ins->log_decoder, NULL, 0); + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); if (cur_buf != data) { flb_free(cur_buf); } @@ -1610,10 +2047,23 @@ int flb_processor_run(struct flb_processor *proc, } if (finalize == FLB_TRUE) { - flb_mp_chunk_cobj_normalize_groups(chunk_cobj); + /* + * The encoder preserves normalized groups and the logical + * count. Reuse them until a filter replaces this serialized + * buffer. + */ + /* keep only the boundary markers the input already had */ + log_buffer_normalized = + flb_mp_chunk_cobj_normalize_groups_ex(chunk_cobj, + chunk_has_group_markers(chunk_cobj) ? + input_group_flags(&group_flags, data, data_size) : 0) == 0; + log_buffer_records = flb_mp_chunk_cobj_count_log_records(chunk_cobj); if (cfl_list_size(&chunk_cobj->records) == 0) { - flb_log_event_encoder_reset(p_ins->log_encoder); + if (output_records) { + *output_records = 0; + } + flb_log_event_encoder_reset(&codec->encoder); flb_mp_chunk_cobj_destroy(chunk_cobj); *out_buf = NULL; @@ -1637,7 +2087,9 @@ int flb_processor_run(struct flb_processor *proc, /* encode chunk_cobj as msgpack */ ret = flb_mp_chunk_cobj_encode(chunk_cobj, (char **) &tmp_buf, &tmp_size); if (ret != 0) { - flb_log_event_decoder_reset(p_ins->log_decoder, NULL, 0); + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); + flb_log_event_encoder_reset(&codec->encoder); + flb_mp_chunk_cobj_destroy(chunk_cobj); if (cur_buf != data) { flb_free(cur_buf); @@ -1667,8 +2119,8 @@ int flb_processor_run(struct flb_processor *proc, cur_size = tmp_size; - flb_log_event_decoder_reset(p_ins->log_decoder, NULL, 0); - flb_log_event_encoder_claim_internal_buffer_ownership(p_ins->log_encoder); + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); + flb_log_event_encoder_claim_internal_buffer_ownership(&codec->encoder); flb_mp_chunk_cobj_destroy(chunk_cobj); chunk_cobj = NULL; } @@ -1677,7 +2129,12 @@ int flb_processor_run(struct flb_processor *proc, unit_out_items = flb_mp_chunk_cobj_count_log_records(chunk_cobj); } else if (cur_buf != NULL) { - unit_out_items = flb_mp_count_log_records(cur_buf, cur_size); + if (log_buffer_records >= 0) { + unit_out_items = log_buffer_records; + } + else { + unit_out_items = flb_mp_count_log_records(cur_buf, cur_size); + } } } } @@ -1704,7 +2161,16 @@ int flb_processor_run(struct flb_processor *proc, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); - out_buf = NULL; + if (cur_buf != data) { + cmt_destroy(cur_buf); + } + + if (out_buf != NULL) { + *out_buf = NULL; + } + if (out_size != NULL) { + *out_size = 0; + } return -1; } @@ -1810,10 +2276,12 @@ int flb_processor_run(struct flb_processor *proc, /* set output buffer */ if (type == FLB_PROCESSOR_LOGS && - cur_buf != NULL) { - ret = flb_mp_normalize_log_buffer_groups_msgpack(cur_buf, cur_size, - (char **) &tmp_buf, - &tmp_size); + cur_buf != NULL && !log_buffer_normalized) { + ret = flb_mp_normalize_log_buffer_groups_msgpack_ex(cur_buf, cur_size, + (char **) &tmp_buf, &tmp_size, + input_group_flags(&group_flags, + data, + data_size)); if (ret == 0) { if (cur_buf != data) { flb_free(cur_buf); @@ -1821,11 +2289,15 @@ int flb_processor_run(struct flb_processor *proc, cur_buf = tmp_buf; cur_size = tmp_size; + log_buffer_records = -1; } } - if (type == FLB_PROCESSOR_LOGS && - (cur_buf == NULL || flb_mp_count_log_records(cur_buf, cur_size) == 0)) { + if (type == FLB_PROCESSOR_LOGS && cur_buf != NULL && log_buffer_records < 0) { + log_buffer_records = flb_mp_count_log_records(cur_buf, cur_size); + } + + if (type == FLB_PROCESSOR_LOGS && (cur_buf == NULL || log_buffer_records == 0)) { if (cur_buf != data) { flb_free(cur_buf); } @@ -1838,6 +2310,10 @@ int flb_processor_run(struct flb_processor *proc, *out_size = 0; } + if (output_records) { + *output_records = 0; + } + return 0; } @@ -1849,9 +2325,33 @@ int flb_processor_run(struct flb_processor *proc, *out_size = cur_size; } + if (output_records && type == FLB_PROCESSOR_LOGS) { + *output_records = log_buffer_records; + } + return 0; } +int flb_processor_run_with_count(struct flb_processor *proc, + size_t starting_stage, + int type, + const char *tag, size_t tag_len, + void *data, size_t data_size, + void **out_buf, size_t *out_size, + int input_records, int *output_records) +{ + int ret; + struct chunk_codec codec; + + codec.ready = FLB_FALSE; + ret = processor_run(proc, &codec, starting_stage, type, tag, tag_len, + data, data_size, out_buf, out_size, + input_records, output_records); + chunk_codec_destroy(&codec); + + return ret; +} + void flb_processor_destroy(struct flb_processor *proc) { struct mk_list *head; diff --git a/src/opentelemetry/flb_opentelemetry_logs.c b/src/opentelemetry/flb_opentelemetry_logs.c index 63687cc833b..6253eb1db74 100644 --- a/src/opentelemetry/flb_opentelemetry_logs.c +++ b/src/opentelemetry/flb_opentelemetry_logs.c @@ -243,8 +243,11 @@ static int process_json_payload_log_records_entry( } } - /* traceFlags */ - result = flb_otel_utils_find_map_entry_by_key(log_records_entry, "traceFlags", 0, FLB_TRUE); + /* Canonical OTLP JSON flags, with legacy traceFlags compatibility. */ + result = flb_otel_utils_find_map_entry_by_key(log_records_entry, "flags", 0, FLB_TRUE); + if (result < 0) { + result = flb_otel_utils_find_map_entry_by_key(log_records_entry, "traceFlags", 0, FLB_TRUE); + } if (result >= 0) { trace_flags = &log_records_entry->ptr[result].val; } @@ -756,6 +759,16 @@ static int process_json_payload_resource_logs_entry (struct flb_log_event_encode } } + /* Preserve the scope's dropped attribute count like the resource's. */ + result = flb_otel_utils_find_map_entry_by_key(&scope->via.map, + "droppedAttributesCount", 0, FLB_TRUE); + if (result >= 0) { + obj = &scope->via.map.ptr[result].val; + flb_log_event_encoder_append_body_values(tmp_encoder, + FLB_LOG_EVENT_CSTRING_VALUE("dropped_attributes_count"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(obj)); + } + /* scope attributes */ result = flb_otel_utils_find_map_entry_by_key(&scope->via.map, "attributes", 0, FLB_TRUE); if (result >= 0) { diff --git a/src/opentelemetry/flb_opentelemetry_metrics.c b/src/opentelemetry/flb_opentelemetry_metrics.c index 61319eec9e2..039b53a17c5 100644 --- a/src/opentelemetry/flb_opentelemetry_metrics.c +++ b/src/opentelemetry/flb_opentelemetry_metrics.c @@ -17,6 +17,7 @@ * limitations under the License. */ +#include #include #include #include @@ -25,6 +26,7 @@ #include #include +#include #include #include #include @@ -320,26 +322,6 @@ static int set_metric_unit(struct cmt_map *map, msgpack_object_map *metric_map) return 0; } -static char *map_type_to_key(int map_type) -{ - switch (map_type) { - case CMT_COUNTER: - return "counter"; - case CMT_GAUGE: - return "gauge"; - case CMT_UNTYPED: - return "untyped"; - case CMT_SUMMARY: - return "summary"; - case CMT_HISTOGRAM: - return "histogram"; - case CMT_EXP_HISTOGRAM: - return "exp_histogram"; - default: - return "unknown"; - } -} - static struct cfl_kvlist *get_or_create_external_metadata_kvlist( struct cfl_kvlist *root, char *key) { @@ -376,62 +358,6 @@ static struct cfl_kvlist *get_or_create_external_metadata_kvlist( * index of the entries created there during the request, keyed by parent * kvlist and lowercased key to keep the case-insensitive kvlist semantics. */ -static struct cfl_kvlist *get_or_create_indexed_kvlist(struct flb_hash_table *index, - struct cfl_kvlist *root, - char *key) -{ - int ret; - size_t i; - size_t out_size; - void *out_buf; - flb_sds_t index_key; - struct cfl_kvlist *entry_kvlist; - - index_key = flb_sds_create_size(strlen(key) + 32); - if (index_key == NULL) { - return NULL; - } - - if (flb_sds_printf(&index_key, "%p:%s", (void *) root, key) == NULL) { - flb_sds_destroy(index_key); - return NULL; - } - - for (i = 0; i < flb_sds_len(index_key); i++) { - index_key[i] = tolower((unsigned char) index_key[i]); - } - - ret = flb_hash_table_get(index, index_key, flb_sds_len(index_key), - &out_buf, &out_size); - if (ret >= 0) { - flb_sds_destroy(index_key); - return (struct cfl_kvlist *) out_buf; - } - - entry_kvlist = cfl_kvlist_create(); - if (entry_kvlist == NULL) { - flb_sds_destroy(index_key); - return NULL; - } - - ret = cfl_kvlist_insert_kvlist(root, key, entry_kvlist); - if (ret != 0) { - cfl_kvlist_destroy(entry_kvlist); - flb_sds_destroy(index_key); - return NULL; - } - - /* the kvlist is owned by root, the index only references it */ - ret = flb_hash_table_add(index, index_key, flb_sds_len(index_key), - entry_kvlist, 0); - flb_sds_destroy(index_key); - if (ret == -1) { - return NULL; - } - - return entry_kvlist; -} - static uint64_t compute_metric_hash(struct cmt_map *map, struct cmt_metric *sample) { struct cfl_list *head; @@ -465,31 +391,10 @@ static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct flb_hash_table *kvlist_index, struct cmt_map *map) { - struct cfl_kvlist *otlp_root; - struct cfl_kvlist *metrics_root; - struct cfl_kvlist *type_root; - - if (cmt == NULL || map == NULL || map->opts == NULL || map->opts->fqname == NULL) { - return NULL; - } - - otlp_root = get_or_create_indexed_kvlist(kvlist_index, cmt->external_metadata, "otlp"); - if (otlp_root == NULL) { - return NULL; + if (!map->otlp_metadata) { + map->otlp_metadata = cfl_kvlist_create(); } - - metrics_root = get_or_create_indexed_kvlist(kvlist_index, otlp_root, "metrics"); - if (metrics_root == NULL) { - return NULL; - } - - type_root = get_or_create_indexed_kvlist(kvlist_index, metrics_root, - map_type_to_key(map->type)); - if (type_root == NULL) { - return NULL; - } - - return get_or_create_indexed_kvlist(kvlist_index, type_root, map->opts->fqname); + return map->otlp_metadata; } static struct cfl_kvlist *get_or_create_data_point_metadata_context( @@ -499,30 +404,81 @@ static struct cfl_kvlist *get_or_create_data_point_metadata_context( struct cmt_metric *sample, uint64_t timestamp) { - char key[128]; - struct cfl_kvlist *metric_context; - struct cfl_kvlist *datapoints_context; - - if (sample != NULL && sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { + if (sample == NULL) { + return NULL; + } + if (sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { sample->hash = compute_metric_hash(map, sample); } + if (sample->otlp_metadata == NULL) { + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata && cmt_otlp_capture_labels(map, sample) != 0) { + return NULL; + } + } + return sample->otlp_metadata; +} - metric_context = get_or_create_metric_metadata_context(cmt, kvlist_index, map); - if (metric_context == NULL) { +/* + * Release the per-sample storage owned by the static (label-less) sample of a + * map and mark it unset. The static sample is embedded in the map, it must + * never be released with cmt_map_metric_destroy(). + */ +static void reset_static_otlp_sample(struct cmt_map *map) +{ + struct cmt_metric *sample; + + sample = &map->metric; + + free(sample->hist_buckets); + free(sample->exp_hist_positive_buckets); + free(sample->exp_hist_negative_buckets); + free(sample->sum_quantiles); + cfl_kvlist_destroy(sample->otlp_metadata); + + sample->hist_buckets = NULL; + sample->exp_hist_positive_buckets = NULL; + sample->exp_hist_negative_buckets = NULL; + sample->sum_quantiles = NULL; + sample->sum_quantiles_count = 0; + sample->otlp_metadata = NULL; + + map->metric_static_set = CMT_FALSE; +} + +/* + * Get (or create) the sample for the given label values. Repeated data points + * of the same series within one request update the same sample, the last data + * point wins. The typed OTLP metadata of the sample is rebuilt so it always + * describes the last data point written to it. + */ +static struct cmt_metric *get_otlp_sample(struct cmt_map *map, int count, char **values) +{ + struct cmt_metric *sample; + + sample = cmt_map_metric_get(map->opts, map, count, values, CMT_TRUE); + if (sample == NULL) { return NULL; } - datapoints_context = get_or_create_indexed_kvlist(kvlist_index, metric_context, - "datapoints"); - if (datapoints_context == NULL) { - return NULL; + if (sample->otlp_metadata != NULL) { + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = NULL; } - snprintf(key, sizeof(key) - 1, "%llx:%llu", - (unsigned long long) (sample != NULL ? sample->hash : 0), - (unsigned long long) timestamp); + sample->otlp_metadata = cfl_kvlist_create(); + if (sample->otlp_metadata == NULL || + cmt_otlp_capture_labels(map, sample) != 0) { + if (sample == &map->metric) { + reset_static_otlp_sample(map); + } + else { + cmt_map_metric_destroy(sample); + } + return NULL; + } - return get_or_create_indexed_kvlist(kvlist_index, datapoints_context, key); + return sample; } static int object_to_sds(msgpack_object *obj, flb_sds_t *out) @@ -1123,6 +1079,7 @@ static void append_common_datapoint_metadata(struct cfl_kvlist *point_metadata, msgpack_object_map *point_map) { int result; + struct cfl_kvlist *attributes; uint64_t start_time_unix_nano; uint64_t flags; msgpack_object *exemplars_object; @@ -1131,6 +1088,16 @@ static void append_common_datapoint_metadata(struct cfl_kvlist *point_metadata, return; } + result = flb_otel_utils_find_map_entry_by_key(point_map, "attributes", 0, FLB_TRUE); + attributes = cfl_kvlist_create(); + if (attributes != NULL) { + if ((result >= 0 && flb_otel_utils_clone_kvlist_from_otlp_json_array( + attributes, &point_map->ptr[result].val) != 0) || + cfl_kvlist_insert_kvlist(point_metadata, "attributes", attributes) != 0) { + cfl_kvlist_destroy(attributes); + } + } + result = parse_optional_datapoint_u64(point_map, "startTimeUnixNano", &start_time_unix_nano); @@ -1577,6 +1544,7 @@ static int process_metric_gauge_data_points(struct cmt *context, return result; } + gauge->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -1591,11 +1559,8 @@ static int process_metric_gauge_data_points(struct cmt *context, } } - sample = cmt_map_metric_get(&gauge->opts, - gauge->map, - point_label_count, - (char **) point_label_values, - CMT_TRUE); + sample = get_otlp_sample(gauge->map, point_label_count, + (char **) point_label_values); if (sample != NULL) { if (number_value_case == CMT_METRIC_VALUE_INT64) { cmt_metric_set_int64(sample, timestamp, int_value); @@ -1771,6 +1736,7 @@ static int process_metric_sum_data_points(struct cmt *context, } counter->aggregation_type = aggregation_type; + counter->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -1785,11 +1751,8 @@ static int process_metric_sum_data_points(struct cmt *context, } } - sample = cmt_map_metric_get(&counter->opts, - counter->map, - point_label_count, - (char **) point_label_values, - CMT_TRUE); + sample = get_otlp_sample(counter->map, point_label_count, + (char **) point_label_values); if (sample != NULL) { if (number_value_case == CMT_METRIC_VALUE_INT64) { if (allow_reset == FLB_FALSE && int_value < 0) { @@ -2176,6 +2139,7 @@ static int process_metric_histogram_data_points(struct cmt *context, uint64_t timestamp; uint64_t count; size_t bucket_counts_count; + size_t bucket_index; size_t metric_bound_count; size_t point_bound_count; double sum; @@ -2466,6 +2430,7 @@ static int process_metric_histogram_data_points(struct cmt *context, } histogram->aggregation_type = aggregation_type; + histogram->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -2492,19 +2457,21 @@ static int process_metric_histogram_data_points(struct cmt *context, } } - cmt_histogram_set_default(histogram, - timestamp, - bucket_counts, - sum, - count, - point_label_count, - (char **) point_label_values); + sample = get_otlp_sample(histogram->map, point_label_count, + (char **) point_label_values); + if (sample != NULL) { + /* CMetrics stores cumulative buckets, including +Inf. */ + for (bucket_index = 1; bucket_index < bucket_counts_count; bucket_index++) { + bucket_counts[bucket_index] += bucket_counts[bucket_index - 1]; + } + /* a repeated series replaces the buckets of the previous point */ + free(sample->hist_buckets); + sample->hist_buckets = bucket_counts; + bucket_counts = NULL; + cmt_metric_hist_sum_set(sample, timestamp, sum); + cmt_metric_hist_count_set(sample, timestamp, count); + } - sample = cmt_map_metric_get(&histogram->opts, - histogram->map, - point_label_count, - (char **) point_label_values, - CMT_FALSE); if (sample != NULL) { point_metadata = get_or_create_data_point_metadata_context(context, kvlist_index, @@ -2544,6 +2511,7 @@ static int process_metric_summary_data_points(struct cmt *context, msgpack_object *name_object, msgpack_object_array *data_points) { + int quantile_index; int index; int result; int metric_initialized; @@ -2758,6 +2726,7 @@ static int process_metric_summary_data_points(struct cmt *context, return result; } + summary->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -2784,19 +2753,36 @@ static int process_metric_summary_data_points(struct cmt *context, } } - cmt_summary_set_default(summary, - timestamp, - point_quantile_values, - sum, - count, - point_label_count, - (char **) point_label_values); + sample = get_otlp_sample(summary->map, point_label_count, + (char **) point_label_values); + if (sample != NULL) { + /* a repeated series reuses the quantile storage of the sample */ + if (sample->sum_quantiles != NULL && + sample->sum_quantiles_count != summary->quantiles_count) { + free(sample->sum_quantiles); + sample->sum_quantiles = NULL; + } + if (sample->sum_quantiles == NULL) { + sample->sum_quantiles = calloc(summary->quantiles_count ? + summary->quantiles_count : 1, + sizeof(uint64_t)); + } + if (sample->sum_quantiles == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + flb_free(point_quantile_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_quantiles); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + sample->sum_quantiles_count = summary->quantiles_count; + cmt_atomic_store(&sample->sum_quantiles_set, CMT_TRUE); + for (quantile_index = 0; quantile_index < (int) summary->quantiles_count; quantile_index++) { + cmt_summary_quantile_set(sample, timestamp, quantile_index, point_quantile_values[quantile_index]); + } + cmt_summary_sum_set(sample, timestamp, sum); + cmt_summary_count_set(sample, timestamp, count); + } - sample = cmt_map_metric_get(&summary->opts, - summary->map, - point_label_count, - (char **) point_label_values, - CMT_FALSE); if (sample != NULL) { point_metadata = get_or_create_data_point_metadata_context(context, kvlist_index, @@ -3144,6 +3130,7 @@ static int process_metric_exponential_histogram_data_points( } exp_histogram->aggregation_type = aggregation_type; + exp_histogram->map->metric_static_set = CMT_FALSE; metric_initialized = FLB_TRUE; } else { @@ -3163,34 +3150,28 @@ static int process_metric_exponential_histogram_data_points( } } - result = cmt_exp_histogram_set_default(exp_histogram, - timestamp, - scale, - zero_count, - zero_threshold, - positive_offset, - positive_count, - positive_counts, - negative_offset, - negative_count, - negative_counts, - sum_set, - sum, - count, - point_label_count, - (char **) point_label_values); - if (result != 0) { - destroy_label_arrays(point_label_count, NULL, point_label_values); - flb_free(positive_counts); - flb_free(negative_counts); - return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + sample = get_otlp_sample(exp_histogram->map, point_label_count, + (char **) point_label_values); + if (sample != NULL) { + sample->exp_hist_scale = scale; + sample->exp_hist_zero_count = zero_count; + sample->exp_hist_zero_threshold = zero_threshold; + sample->exp_hist_positive_offset = positive_offset; + sample->exp_hist_negative_offset = negative_offset; + sample->exp_hist_positive_count = positive_count; + sample->exp_hist_negative_count = negative_count; + /* a repeated series replaces the buckets of the previous point */ + free(sample->exp_hist_positive_buckets); + free(sample->exp_hist_negative_buckets); + sample->exp_hist_positive_buckets = positive_counts; + sample->exp_hist_negative_buckets = negative_counts; + positive_counts = NULL; + negative_counts = NULL; + cmt_metric_set_exp_hist_count(sample, count); + cmt_metric_set_exp_hist_sum(sample, sum_set, sum); + cmt_metric_set_timestamp(sample, timestamp); } - sample = cmt_map_metric_get(&exp_histogram->opts, - exp_histogram->map, - point_label_count, - (char **) point_label_values, - CMT_FALSE); if (sample != NULL) { point_metadata = get_or_create_data_point_metadata_context(context, kvlist_index, diff --git a/src/opentelemetry/flb_opentelemetry_otlp_json.c b/src/opentelemetry/flb_opentelemetry_otlp_json.c index 150dcff2aad..316c2393a3b 100644 --- a/src/opentelemetry/flb_opentelemetry_otlp_json.c +++ b/src/opentelemetry/flb_opentelemetry_otlp_json.c @@ -17,6 +17,7 @@ * limitations under the License. */ +#include #include #include #include @@ -1541,6 +1542,25 @@ static struct flb_json_mut_val *log_record_to_json(struct flb_json_mut_doc *doc, } } + if (otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP && + otlp_uint64_field_value(&otlp_metadata->via.map, "dropped_attributes_count", ×tamp) == 0 && + !flb_json_mut_obj_add_uint(doc, record, "droppedAttributesCount", timestamp)) { + return NULL; + } + field = otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP ? + msgpack_map_get_object(&otlp_metadata->via.map, "trace_flags") : NULL; + if (field != NULL && field->type == MSGPACK_OBJECT_POSITIVE_INTEGER && + !flb_json_mut_obj_add_uint(doc, record, "flags", field->via.u64)) { + return NULL; + } + + field = otlp_metadata != NULL && otlp_metadata->type == MSGPACK_OBJECT_MAP ? + msgpack_map_get_object(&otlp_metadata->via.map, "event_name") : NULL; + if (field != NULL && field->type == MSGPACK_OBJECT_STR && + !flb_json_mut_obj_add_strncpy(doc, record, "eventName", field->via.str.ptr, field->via.str.size)) { + return NULL; + } + body = extract_log_body_value(doc, event->body, logs_body_keys, @@ -2068,6 +2088,10 @@ static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, return NULL; } + if (map->otlp_metadata) { + return map->otlp_metadata; + } + otlp_root = fetch_metadata_kvlist_key(cmt->external_metadata, "otlp"); metrics_root = fetch_metadata_kvlist_key(otlp_root, "metrics"); type_root = fetch_metadata_kvlist_key(metrics_root, metrics_map_type_to_key(map->type)); @@ -2087,6 +2111,10 @@ static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, struct cfl_kvlist *metric_context; struct cfl_kvlist *datapoints_context; + if (sample != NULL && sample->otlp_metadata != NULL) { + return sample->otlp_metadata; + } + metric_context = get_metric_otlp_metadata_context(cmt, map); datapoints_context = fetch_metadata_kvlist_key(metric_context, "datapoints"); @@ -2131,7 +2159,10 @@ static size_t resolve_target_scope_index(struct cmt *cmt, struct cfl_kvlist *metric_context; metric_context = get_metric_otlp_metadata_context(cmt, map); - metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + metadata = fetch_metadata_kvlist_key(metric_context, "routing"); + if (metadata == NULL) { + metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + } if (metadata == NULL) { return 0; @@ -2397,6 +2428,7 @@ static int add_data_point_common_fields(struct flb_json_mut_doc *doc, struct cmt_map_label *label_name; struct cmt_map_label *label_value; struct cfl_kvlist *metadata; + struct cfl_kvlist *typed_attributes; struct cfl_variant *variant; struct cfl_array *array; struct flb_json_mut_val *attributes; @@ -2404,7 +2436,13 @@ static int add_data_point_common_fields(struct flb_json_mut_doc *doc, struct flb_json_mut_val *value; struct flb_json_mut_val *exemplars; - attributes = flb_json_mut_arr(doc); + metadata = get_data_point_otlp_metadata_context(cmt, map, sample); + typed_attributes = fetch_metadata_kvlist_key(metadata, "attributes"); + if (!cmt_otlp_attributes_match_labels(map, sample)) { + typed_attributes = NULL; + } + attributes = typed_attributes ? cfl_kvlist_to_otlp_kv_array(doc, typed_attributes) : + flb_json_mut_arr(doc); if (attributes == NULL) { return -1; } @@ -2429,6 +2467,7 @@ static int add_data_point_common_fields(struct flb_json_mut_doc *doc, label_index = 0; label_name = cfl_list_entry_first(&map->label_keys, struct cmt_map_label, _head); + if (typed_attributes == NULL) { cfl_list_foreach(head, &sample->labels) { label_value = cfl_list_entry(head, struct cmt_map_label, _head); attribute = flb_json_mut_obj(doc); @@ -2454,6 +2493,8 @@ static int add_data_point_common_fields(struct flb_json_mut_doc *doc, &map->label_keys); } + } + if (flb_json_mut_arr_size(attributes) > 0 && !flb_json_mut_obj_add_val(doc, point, "attributes", attributes)) { return -1; @@ -2697,7 +2738,7 @@ static struct flb_json_mut_val *create_histogram_data_point_json(struct flb_json int length; length = snprintf(buffer, sizeof(buffer), "%" PRIu64, - sample->hist_buckets[index]); + cmt_metric_hist_get_bucket_count(sample, index)); if (length <= 0 || (size_t) length >= sizeof(buffer) || !flb_json_mut_arr_add_strncpy(doc, bucket_counts, buffer, length)) { return NULL; @@ -3628,8 +3669,7 @@ static struct flb_json_mut_val *create_trace_span_json(struct flb_json_mut_doc * return NULL; } - if (span->kind >= 0 && - !flb_json_mut_obj_add_int(doc, json, "kind", span->kind)) { + if (!flb_json_mut_obj_add_int(doc, json, "kind", span->kind)) { return NULL; } diff --git a/src/opentelemetry/flb_opentelemetry_traces.c b/src/opentelemetry/flb_opentelemetry_traces.c index ad5f3118b93..56b4e185e68 100644 --- a/src/opentelemetry/flb_opentelemetry_traces.c +++ b/src/opentelemetry/flb_opentelemetry_traces.c @@ -26,194 +26,21 @@ #include -static int process_attribute(struct ctrace_attributes *attr, - msgpack_object *key, msgpack_object *value, int type) -{ - int ret; - char *key_str; - char *value_str; - int64_t value_int; - double value_double; - int value_bool; - - if (key->type != MSGPACK_OBJECT_STR) { - return -1; - } - - /* temporary buffer for key since it needs to be NULL terminated */ - key_str = flb_sds_create_len(key->via.str.ptr, key->via.str.size); - if (key_str == NULL) { - return -1; - } - - /* - * the value of 'type' is set by the JSON wrapped value, we need to to convert it - * since msgpack_object *value is always a string - */ - switch (type) { - case MSGPACK_OBJECT_STR: - if (value->type != MSGPACK_OBJECT_STR) { - flb_sds_destroy(key_str); - return -1; - } - - value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); - if (value_str == NULL) { - flb_sds_destroy(key_str); - return -1; - } - - ret = ctr_attributes_set_string(attr, key_str, value_str); - flb_sds_destroy(value_str); - break; - case MSGPACK_OBJECT_POSITIVE_INTEGER: - case MSGPACK_OBJECT_NEGATIVE_INTEGER: - if (value->type == MSGPACK_OBJECT_STR) { - value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); - if (value_str == NULL) { - flb_sds_destroy(key_str); - return -1; - } - value_int = strtoll(value_str, NULL, 10); - flb_sds_destroy(value_str); - } - else if (value->type == MSGPACK_OBJECT_POSITIVE_INTEGER || - value->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - value_int = value->via.i64; - } - else { - flb_sds_destroy(key_str); - return -1; - } - - ret = ctr_attributes_set_int64(attr, key_str, value_int); - break; - case MSGPACK_OBJECT_FLOAT32: - case MSGPACK_OBJECT_FLOAT64: - if (value->type == MSGPACK_OBJECT_STR) { - value_str = flb_sds_create_len(value->via.str.ptr, value->via.str.size); - if (value_str == NULL) { - flb_sds_destroy(key_str); - return -1; - } - value_double = strtod(value_str, NULL); - flb_sds_destroy(value_str); - } - else if (value->type == MSGPACK_OBJECT_FLOAT32 || - value->type == MSGPACK_OBJECT_FLOAT64) { - value_double = value->via.f64; - } - else { - flb_sds_destroy(key_str); - return -1; - } - - ret = ctr_attributes_set_double(attr, key_str, value_double); - break; - case MSGPACK_OBJECT_BOOLEAN: - if (value->type != MSGPACK_OBJECT_BOOLEAN) { - flb_sds_destroy(key_str); - return -1; - } - - value_bool = value->via.boolean; - ret = ctr_attributes_set_bool(attr, key_str, value_bool); - break; - case MSGPACK_OBJECT_ARRAY: - /* - * The less fun part (OTel JSON encoding), the value can be an array and - * only allows values such as string, bool, int64, double. I am glad this - * don't support nested arrays or maps. - */ - ret = 0; - break; - default: - flb_sds_destroy(key_str); - return -1; - } - - flb_sds_destroy(key_str); - return ret; -} - -static int process_resource_unwrap_attribute(msgpack_object *attr, - msgpack_object *out_key, - msgpack_object *out_value, int *out_value_type) -{ - int ret; - int type; - msgpack_object key; - msgpack_object val; - msgpack_object *real_value; - - if (attr->type != MSGPACK_OBJECT_MAP) { - return -1; - } - - ret = flb_otel_utils_find_map_entry_by_key(&attr->via.map, "key", 0, FLB_TRUE); - if (ret == -1) { - return -1; - } - - key = attr->via.map.ptr[ret].val; - if (key.type != MSGPACK_OBJECT_STR) { - return -1; - } - - ret = flb_otel_utils_find_map_entry_by_key(&attr->via.map, "value", 0, FLB_TRUE); - if (ret == -1) { - return -1; - } - - val = attr->via.map.ptr[ret].val; - - ret = flb_otel_utils_json_payload_get_wrapped_value(&val, &real_value, &type); - if (ret != 0) { - return -1; - } - - *out_key = key; - *out_value = *real_value; - *out_value_type = type; - - return 0; -} - -/* - * Convert a list of attributes in msgpack format to a cfl attributes by - * unwrapping JSON encoded value types. - */ +/* Use the shared AnyValue decoder for scalars, bytes, arrays and nested maps. */ static struct ctrace_attributes *convert_attributes(msgpack_object *attributes, const char *log_context) { - int i; - int ret; - int value_type; - msgpack_object key; - msgpack_object value; struct ctrace_attributes *attr; attr = ctr_attributes_create(); - if (attr == NULL) { + if (!attr) { return NULL; } - - for (i = 0; i < attributes->via.array.size; i++) { - ret = process_resource_unwrap_attribute(&attributes->via.array.ptr[i], - &key, &value, &value_type); - if (ret == -1) { - flb_warn("found invalid %s attribute, skipping", log_context); - continue; - } - - /* set attribute */ - ret = process_attribute(attr, &key, &value, value_type); - if (ret == -1) { - flb_warn("failed to set %s attribute, skipping", log_context); - continue; - } + if (flb_otel_utils_clone_kvlist_from_otlp_json_array(attr->kv, attributes) != 0) { + flb_warn("failed to decode %s attributes", log_context); + ctr_attributes_destroy(attr); + return NULL; } - return attr; } @@ -854,8 +681,16 @@ static int process_spans(struct ctrace *ctr, /* kind */ ret = flb_otel_utils_find_map_entry_by_key(&span.via.map, "kind", 0, FLB_TRUE); - if (ret >= 0 && span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - ctr_span_kind_set(ctr_span, span.via.map.ptr[ret].val.via.u64); + if (ret >= 0) { + /* Proto3 enums preserve unknown numeric values within int32. */ + if (span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER && + span.via.map.ptr[ret].val.via.u64 <= INT32_MAX) { + ctr_span->kind = (int32_t) span.via.map.ptr[ret].val.via.u64; + } + else if (span.via.map.ptr[ret].val.type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + span.via.map.ptr[ret].val.via.i64 >= INT32_MIN) { + ctr_span->kind = (int32_t) span.via.map.ptr[ret].val.via.i64; + } } /* attributes */ diff --git a/src/opentelemetry/flb_opentelemetry_utils.c b/src/opentelemetry/flb_opentelemetry_utils.c index f6e97620c3b..2eea40a04db 100644 --- a/src/opentelemetry/flb_opentelemetry_utils.c +++ b/src/opentelemetry/flb_opentelemetry_utils.c @@ -26,6 +26,76 @@ #include #include +#include + +/* + * OTLP/JSON encodes bytesValue as base64 (standard or URL safe alphabet, with + * or without padding). Decode it into a newly allocated buffer owned by the + * caller. Returns 0 on success and -1 for invalid input or allocation errors. + */ +static int otlp_json_bytes_decode(const char *src, size_t src_size, + unsigned char **out_buf, size_t *out_size) +{ + int ret; + char c; + char *normalized; + size_t index; + size_t padded_size; + size_t decoded_size; + unsigned char *decoded; + + *out_buf = NULL; + *out_size = 0; + + if (src_size % 4 == 1) { + return -1; + } + + padded_size = src_size + ((4 - (src_size % 4)) % 4); + + normalized = flb_malloc(padded_size + 1); + if (normalized == NULL) { + flb_errno(); + return -1; + } + + for (index = 0; index < src_size; index++) { + c = src[index]; + if (c == '-') { + c = '+'; + } + else if (c == '_') { + c = '/'; + } + normalized[index] = c; + } + + for (; index < padded_size; index++) { + normalized[index] = '='; + } + normalized[padded_size] = '\0'; + + decoded = flb_malloc((padded_size / 4) * 3 + 1); + if (decoded == NULL) { + flb_errno(); + flb_free(normalized); + return -1; + } + + ret = flb_base64_decode(decoded, (padded_size / 4) * 3 + 1, &decoded_size, + (unsigned char *) normalized, padded_size); + flb_free(normalized); + + if (ret != 0) { + flb_free(decoded); + return -1; + } + + *out_buf = decoded; + *out_size = decoded_size; + + return 0; +} int flb_otel_utils_find_map_entry_by_key(msgpack_object_map *map, char *key, @@ -132,7 +202,8 @@ int flb_otel_utils_json_payload_get_wrapped_value(msgpack_object *wrapper, internal_type = MSGPACK_OBJECT_FLOAT; } else if (strncasecmp(kv_key->ptr, "bytesValue", kv_key->size) == 0) { - if (kv_value->type != MSGPACK_OBJECT_BIN) { + if (kv_value->type != MSGPACK_OBJECT_BIN && + kv_value->type != MSGPACK_OBJECT_STR) { /* If the value is not binary, we cannot process it */ return -2; } @@ -300,6 +371,8 @@ int flb_otel_utils_json_payload_append_unwrapped_value( char temporary_buffer[33]; int unwrap_value; int result; + size_t bytes_size; + unsigned char *bytes; msgpack_object *value; int type; @@ -344,7 +417,19 @@ int flb_otel_utils_json_payload_append_unwrapped_value( unwrap_value = FLB_TRUE; } else if (type == MSGPACK_OBJECT_BIN) { - unwrap_value = FLB_TRUE; + if (value->type == MSGPACK_OBJECT_STR) { + if (otlp_json_bytes_decode(value->via.str.ptr, value->via.str.size, + &bytes, &bytes_size) != 0) { + return -2; + } + + result = flb_log_event_encoder_append_binary(encoder, target_field, + (char *) bytes, bytes_size); + flb_free(bytes); + } + else { + unwrap_value = FLB_TRUE; + } } else if (type == MSGPACK_OBJECT_ARRAY) { @@ -694,58 +779,6 @@ uint64_t flb_otel_utils_convert_string_number_to_u64(char *str, size_t len) return val; } -static int parse_u64_from_numeric_string(const char *str, - size_t len, - uint64_t *value) -{ - size_t index; - - if (str == NULL || value == NULL || len == 0) { - return -1; - } - - if (str[0] == '-' || len > 31) { - return -1; - } - - for (index = 0; index < len; index++) { - if (!isdigit((unsigned char) str[index])) { - return -1; - } - } - - *value = flb_otel_utils_convert_string_number_to_u64((char *) str, len); - - return 0; -} - -static int parse_u64_from_msgpack_object(msgpack_object *object, uint64_t *value) -{ - if (object == NULL || value == NULL) { - return -1; - } - - if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - *value = object->via.u64; - return 0; - } - else if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { - if (object->via.i64 < 0) { - return -1; - } - - *value = (uint64_t) object->via.i64; - return 0; - } - else if (object->type == MSGPACK_OBJECT_STR) { - return parse_u64_from_numeric_string(object->via.str.ptr, - object->via.str.size, - value); - } - - return -1; -} - static int parse_double_from_msgpack_object(msgpack_object *object, double *value) { char *end; @@ -790,6 +823,9 @@ static int parse_double_from_msgpack_object(msgpack_object *object, double *valu return 0; } +static struct cfl_variant *convert_otlp_anyvalue_wrapper_to_cfl_variant( + msgpack_object *wrapper_object); + static int clone_kvlist_entry_from_otlp_json(struct cfl_kvlist *target, msgpack_object *entry_object) { @@ -821,7 +857,7 @@ static int clone_kvlist_entry_from_otlp_json(struct cfl_kvlist *target, return -1; } - value_instance = flb_otel_utils_msgpack_object_to_cfl_variant(value_object); + value_instance = convert_otlp_anyvalue_wrapper_to_cfl_variant(value_object); if (value_instance == NULL) { return -1; } @@ -837,12 +873,58 @@ static int clone_kvlist_entry_from_otlp_json(struct cfl_kvlist *target, return 0; } +static int parse_i64_from_msgpack_object(msgpack_object *object, int64_t *value) +{ + uint64_t magnitude = 0; + uint64_t limit; + size_t index = 0; + unsigned int digit; + int negative = 0; + + if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *value = object->via.i64; + return 0; + } + if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (object->via.u64 > INT64_MAX) { + return -1; + } + *value = (int64_t) object->via.u64; + return 0; + } + if (object->type != MSGPACK_OBJECT_STR || object->via.str.size == 0) { + return -1; + } + if (object->via.str.ptr[0] == '-') { + negative = 1; + index = 1; + } + if (index == object->via.str.size) { + return -1; + } + limit = (uint64_t) INT64_MAX + negative; + for (; index < object->via.str.size; index++) { + if (object->via.str.ptr[index] < '0' || object->via.str.ptr[index] > '9') { + return -1; + } + digit = object->via.str.ptr[index] - '0'; + if (magnitude > (limit - digit) / 10) { + return -1; + } + magnitude = magnitude * 10 + digit; + } + *value = negative ? (magnitude == (uint64_t) INT64_MAX + 1 ? INT64_MIN : + -(int64_t) magnitude) : (int64_t) magnitude; + return 0; +} + static struct cfl_variant *convert_otlp_anyvalue_wrapper_to_cfl_variant( msgpack_object *wrapper_object) { int value_type; int64_t signed_value; - uint64_t unsigned_value; + unsigned char *bytes; + size_t bytes_size; double double_value; size_t index; msgpack_object *value_object; @@ -883,19 +965,10 @@ static struct cfl_variant *convert_otlp_anyvalue_wrapper_to_cfl_variant( return cfl_variant_create_from_bool(value_object->via.boolean); } else if (value_type == MSGPACK_OBJECT_POSITIVE_INTEGER) { - if (parse_u64_from_msgpack_object(value_object, &unsigned_value) != 0) { + if (parse_i64_from_msgpack_object(value_object, &signed_value) != 0) { return NULL; } - - if (value_object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER || - (value_object->type == MSGPACK_OBJECT_STR && - value_object->via.str.size > 0 && - value_object->via.str.ptr[0] == '-')) { - signed_value = (int64_t) unsigned_value; - return cfl_variant_create_from_int64(signed_value); - } - - return cfl_variant_create_from_uint64(unsigned_value); + return cfl_variant_create_from_int64(signed_value); } else if (value_type == MSGPACK_OBJECT_FLOAT) { if (parse_double_from_msgpack_object(value_object, &double_value) != 0) { @@ -905,6 +978,17 @@ static struct cfl_variant *convert_otlp_anyvalue_wrapper_to_cfl_variant( return cfl_variant_create_from_double(double_value); } else if (value_type == MSGPACK_OBJECT_BIN) { + if (value_object->type == MSGPACK_OBJECT_STR) { + if (otlp_json_bytes_decode(value_object->via.str.ptr, + value_object->via.str.size, + &bytes, &bytes_size) != 0) { + return NULL; + } + result_variant = cfl_variant_create_from_bytes((char *) bytes, bytes_size, + CFL_FALSE); + flb_free(bytes); + return result_variant; + } if (value_object->type != MSGPACK_OBJECT_BIN) { return NULL; } @@ -1134,6 +1218,8 @@ int flb_otel_utils_clone_kvlist_from_otlp_json_array(struct cfl_kvlist *target, msgpack_object *attributes_object) { size_t index; + size_t invalid_count; + size_t first_invalid; msgpack_object_array *attributes_array; if (target == NULL || attributes_object == NULL) { @@ -1145,13 +1231,26 @@ int flb_otel_utils_clone_kvlist_from_otlp_json_array(struct cfl_kvlist *target, } attributes_array = &attributes_object->via.array; + invalid_count = 0; + first_invalid = 0; for (index = 0 ; index < attributes_array->size ; index++) { if (clone_kvlist_entry_from_otlp_json(target, &attributes_array->ptr[index]) != 0) { - return -1; + /* An invalid attribute must not discard its healthy siblings. */ + if (invalid_count == 0) { + first_invalid = index; + } + invalid_count++; } } + /* report once per attribute list to avoid flooding the log */ + if (invalid_count > 0) { + flb_warn("ignoring %zu invalid OTLP JSON attribute(s) out of %zu " + "(first at index %zu)", + invalid_count, (size_t) attributes_array->size, first_invalid); + } + return 0; } diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_batch_size.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_batch_size.yaml new file mode 100644 index 00000000000..0fdad8c9533 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_batch_size.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + batch_size: 3 diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_groups.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_groups.yaml new file mode 100644 index 00000000000..2d327081491 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_forward_groups.yaml @@ -0,0 +1,17 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py b/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py index 0481f64b65c..647131c6720 100644 --- a/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py +++ b/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py @@ -6,6 +6,7 @@ import shutil import socket import threading +import time from collections import Counter import requests @@ -1454,3 +1455,91 @@ def test_out_opentelemetry_logs_max_scopes_enforcement(): output = json.loads(json_format.MessageToJson(logs_seen[0])) assert len(output["resourceLogs"]) == 4 assert all(len(resource_log["scopeLogs"]) == 1 for resource_log in output["resourceLogs"]) + + +def _pack(value): + """Encode forward fixtures without an optional MessagePack dependency.""" + if isinstance(value, bytes): + return value + if isinstance(value, int): + return b"\xcf" + value.to_bytes(8, "big") + if isinstance(value, str): + data = value.encode("utf-8") + return b"\xdb" + len(data).to_bytes(4, "big") + data + if isinstance(value, list): + return b"\xdd" + len(value).to_bytes(4, "big") + b"".join(_pack(item) for item in value) + if isinstance(value, dict): + return b"\xdf" + len(value).to_bytes(4, "big") + b"".join( + _pack(key) + _pack(item) for key, item in value.items()) + raise TypeError(type(value)) + + +def _event_time(seconds): + return b"\xd7\x00" + seconds.to_bytes(4, "big") + bytes(4) + + +def _group_start(resource_id, name): + return [[_event_time(0xFFFFFFFF), {"schema": "otlp", "resource_id": resource_id, "scope_id": 0}], + {"resource": {"attributes": {"service.name": name}}, "scope": {"name": "scope"}}] + + +def _group_end(): + return [[_event_time(0xFFFFFFFE), {}], {}] + + +def _log_record(record_id): + return [[_event_time(1700000000), {"otlp": {}}], {"id": record_id}] + + +def _send_forward(service, entries): + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=10) as connection: + connection.sendall(_pack(["test", entries])) + + +def _exported_record_ids(): + ids = [] + for export_request in data_storage["logs"]: + output = json.loads(json_format.MessageToJson(export_request)) + for record, _, _ in iter_log_records(output): + for item in record["body"]["kvlistValue"]["values"]: + if item["key"] == "id": + ids.append(int(item["value"]["intValue"])) + return sorted(ids) + + +def test_out_opentelemetry_chunk_ending_with_empty_group_exports_its_records(): + """A chunk that ends with a group start and no records still exports the earlier groups.""" + service = Service("out_otel_http_logs_forward_groups.yaml") + service.start() + try: + # the next group starts in this append and receives its records later + _send_forward(service, [_group_start(0, "first"), _log_record(0), _log_record(1), + _group_end(), _group_start(1, "second")]) + service.service.wait_for_condition( + lambda: _exported_record_ids() == [0, 1] or None, + timeout=15, interval=0.5, description="records of the first group") + + _send_forward(service, [_log_record(2), _group_end()]) + service.service.wait_for_condition( + lambda: _exported_record_ids() == [0, 1, 2] or None, + timeout=15, interval=0.5, description="record appended to the open group") + finally: + service.stop() + + +def test_out_opentelemetry_batch_size_exports_each_record_once(): + """Records of earlier scopes are not sent again by a later batch.""" + service = Service("out_otel_http_logs_forward_batch_size.yaml") + service.start() + try: + entries = [_group_start(0, "first")] + [_log_record(i) for i in range(2)] + [_group_end()] + entries += [_group_start(1, "second")] + [_log_record(i) for i in range(2, 7)] + [_group_end()] + _send_forward(service, entries) + service.service.wait_for_condition( + lambda: len(_exported_record_ids()) >= 7 or None, + timeout=15, interval=0.5, description="seven exported records") + # give a duplicated export time to arrive + time.sleep(2) + assert _exported_record_ids() == list(range(7)) + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts.py b/tests/integration/scenarios/processor_fts/tests/test_fts.py new file mode 100644 index 00000000000..578303900e0 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts.py @@ -0,0 +1,346 @@ +"""FTS (Fluent Typed Scripting): ownership, exact values, and OTel groups.""" + +import json +import re +import requests +from pathlib import Path + +import pytest +import yaml + +from utils.fluent_bit_manager import FluentBitStartupError +from utils.test_service import FluentBitTestService + + +def make_service(tmp_path, code, *, grouped=False, error_mode="keep", + threaded=False, output_processor=False, second_processor=None, output_format="json_lines", runtime_options=None): + processor = {"name": "fts", "code": code, "call": "process_log", + "error_mode": error_mode} + processor.update(runtime_options or {}) + processors = ([{"name": "opentelemetry_envelope"}] if grouped else []) + [processor] + if second_processor: + processors.append(second_processor) + dummy = {"name": "dummy", "tag": "typed.logs", "rate": 4, "threaded": threaded, + "dummy": json.dumps({"message": "original", "secret": "sensitive", + "Secret": "case-sensitive", "optional": None, + "large": 18446744073709551615, + "nested": {"one": 1}, "values": [1, 2]})} + output = {"name": "stdout", "match": "*", "format": output_format} + (output if output_processor else dummy)["processors"] = {"logs": processors} + config = tmp_path / "typed.yaml" + config.write_text(yaml.safe_dump({ + "service": {"flush": 0.2, "grace": 1, "log_level": "info", "http_server": True, + "http_listen": "127.0.0.1", + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [dummy], "outputs": [output]}, + })) + return FluentBitTestService(str(config)) + + +def log_text(service): + return Path(service.flb.log_file).read_text(encoding="utf-8") + + +def records(service): + return [json.loads(line) for line in log_text(service).splitlines() if line.startswith("{")] + + +@pytest.mark.parametrize("output_processor", [False, True]) +def test_removed_libraries_cannot_be_recovered(tmp_path, output_processor): + code = r''' +local function check_packages() + for _, name in ipairs({"os", "io", "debug", "jit", "ffi", "package", "coroutine"}) do + assert(rawget(_G, name) == nil) + assert(getfenv == nil and setfenv == nil) + assert(not pcall(require, name)) + assert(not pcall(function() module(name) end)) + end + assert(module == nil and loadfile == nil and dofile == nil) + assert(not pcall(require, "fluentbit\0os")) + assert(type(require("fluentbit")) == "table") +end +check_packages() +function process_log(log) + check_packages() + log.body:set("message", "restricted") +end +''' + service = make_service(tmp_path, code, output_processor=output_processor) + try: + service.start() + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(record["message"] == "restricted" for record in output) + assert "edits rolled back" not in log_text(service) + finally: + service.stop() + + +TYPED_CODE = r''' +local fb = require("fluentbit") +local previous, retained +function process_log(event) + local body = event.body + assert(body:get_string("message") == "original") + assert(body:get_string("missing") == nil) + assert(not pcall(function() body:get_number("values") end)) + assert(body:get("large"):type() == "uint64") + assert(body:get("large"):value() == "18446744073709551615") + assert(not pcall(function() return body:get("large"):number() end)) + assert(body:get("optional"):type() == "null") + assert(body:get("missing") == nil) + body:set("copied", body:get("large")) + body:set("signed", fb.int64("-9223372036854775808")) + body:set("unsigned", fb.uint64("18446744073709551615")) + body:set("null", fb.null) + body:set("empty_map", fb.map()) + body:set("empty_array", fb.array()) + body:set("flag", true) + assert(body:get_bool("flag") == true) + body:set("fraction", 1.25) + assert(body:get_number("fraction") == 1.25) + local allowed = fb.map() + allowed:set("keep", true) + allowed:set("remove", true) + fb.keep_keys(allowed, {"keep"}) + assert(allowed:len() == 1 and allowed:get("remove") == nil) + fb.redact(allowed, {"keep"}, false) + assert(allowed:get("keep"):value() == false) + allowed:set("nul\0key", "nul\0value") + assert(allowed:get("nul\0key"):value() == "nul\0value") + assert(not pcall(function() fb.uint64(" 1") end)) + assert(not pcall(function() fb.uint64("1\0") end)) + assert(not pcall(function() fb.int64(9007199254740992) end)) + assert(not pcall(function() fb.attributes(event) end)) + body:set("utf8", "café") + body:get("nested"):set("one", fb.int64(7)) + local child = body:get("nested"):get("one") + body:remove("nested") + assert(not pcall(function() return child:value() end)) + body:set("nested", fb.map()) + body:get("nested"):set("one", fb.int64(8)) + assert(child:number() == 8) -- Views resolve paths after replacement. + body:get("values"):set(1, fb.map()) + body:get("values"):get(1):set("temporary", true) + body:get("values"):set(1, fb.int64(3)) + body:get("values"):append(fb.int64(4)) + assert(body:get("values"):len() == 3) + local binary = fb.bytes("a\0b") + assert(binary:type() == "bytes" and binary:value() == "a\0b") + body:set("binary", binary) + fb.redact(body, {"secret"}) + assert(body:get("Secret"):value() == "[REDACTED]") + if previous then + -- Copies are owned and cannot change already committed records. + previous:set("message", "late mutation") + assert(retained:value() == "18446744073709551615") + end + previous, retained = body:copy(), body:get("large"):copy() + body:set("message", "typed") + event.timestamp.sec, event.timestamp.nsec = 2147483648, 123456789 + return fb.KEEP +end +''' + + +@pytest.mark.parametrize("threaded,output_processor", [(False, False), (True, False), (False, True)]) +def test_typed_values_and_ownership(tmp_path, threaded, output_processor): + service = make_service(tmp_path, TYPED_CODE, threaded=threaded, output_processor=output_processor) + try: + service.start() + output = service.wait_for_condition(lambda: records(service) if len(records(service)) >= 3 else None, + timeout=30) + assert "callback failed" not in log_text(service) + for record in output: + assert record["message"] == "typed" + assert record["large"] == record["copied"] == record["unsigned"] == 18446744073709551615 + assert record["signed"] == -9223372036854775808 + assert record["null"] is None and record["optional"] is None + assert record["empty_map"] == {} and record["empty_array"] == [] + assert record["nested"] == {"one": 8} and record["values"] == [3, 2, 4] + assert record["secret"] == "[REDACTED]" and record["Secret"] == "[REDACTED]" + assert record["flag"] is True and record["fraction"] == 1.25 and record["utf8"] == "café" + finally: + service.stop() + + +@pytest.mark.parametrize("failure", [ + 'error("intentional")', + 'do return "invalid-result" end', + 'event.timestamp.nsec = 1000000000', + 'event.metadata = fb.array()', + 'event.body:set("bad", nil)', + 'fb.uint64("18446744073709551616")', + 'fb.int64("-9223372036854775809")', + 'fb.uint64(-1)', + 'event.body:get("values"):get(0)', +]) +def test_errors_rollback_record(tmp_path, failure): + code = f'''local fb = require("fluentbit") +function process_log(event) + event.body:set("message", "partial edit") + {failure} + return fb.KEEP +end''' + service = make_service(tmp_path, code) + try: + service.start() + service.wait_for_condition(lambda: "edits rolled back" in log_text(service), timeout=30) + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(record["message"] == "original" for record in output) + def errors_visible(): + response = requests.get(f"http://127.0.0.1:{service.flb.http_monitoring_port}/api/v2/metrics/prometheus", + timeout=5) + matches = re.findall(r'^fluentbit_processor_errors_total\{[^}]*processor="fts"[^}]*\} ([0-9.eE+-]+)', + response.text, re.MULTILINE) + return any(float(value) > 0 for value in matches) + service.wait_for_condition(errors_visible, timeout=30) + finally: + service.stop() + + +@pytest.mark.parametrize("explicit", [True, False]) +def test_record_drop(tmp_path, explicit): + code = '''local fb = require("fluentbit") +function process_log(event) + %s +end''' % ('return fb.DROP' if explicit else 'error("drop-on-error")') + service = make_service(tmp_path, code, grouped=True, error_mode="drop") + try: + service.start() + service.stop() + assert not records(service) + if not explicit: + assert "edits rolled back" in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("code", ["function !bad syntax", 'error("initializer-failed")', "function other() end"]) +def test_startup_failure_cleanup(tmp_path, code): + service = make_service(tmp_path, code) + try: + with pytest.raises(FluentBitStartupError): + service.start() + assert "FTS initialization failed" in log_text(service) + finally: + service.stop() + + +def test_context_aliases_survive_processor_chain(tmp_path): + code = '''local fb = require("fluentbit") +function process_log(log) + log.body:set("large", fb.int64("9223372036854775807")) + log.resource.attributes:set("service.name", "payments") + log.scope.name = "policy" + log.scope.version = "2" + log.scope.attributes:set("revision", fb.int64(3)) + log.attributes:set("redacted", true) + fb.redact(log.body, {"secret"}) +end''' + downstream = {"name": "fts", "code": '''function process(log) + assert(log.resource.attributes:get_string("service.name") == "payments") + assert(log.scope.name == "policy" and log.scope.version == "2") + assert(log.scope.attributes:get_number("revision") == 3) + assert(log.attributes:get_bool("redacted")) + log.body:set("verified", true) + end'''} + service = make_service(tmp_path, code, grouped=True, second_processor=downstream) + try: + service.start() + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(record["verified"] and record["secret"] == "[REDACTED]" for record in output) + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +def test_script_file_and_default_callback(tmp_path): + script = tmp_path / "policy.lua" + script.write_text('local fb=require("fluentbit"); function process(log) ' + 'log.body:set("message", "script-file"); log.body:set("count",fb.int64(1)) end') + service = make_service(tmp_path, "unused") + config = tmp_path / "typed.yaml" + document = yaml.safe_load(config.read_text()) + processor = document["pipeline"]["inputs"][0]["processors"]["logs"][0] + processor.pop("call"); processor.pop("code"); processor["script"] = str(script) + config.write_text(yaml.safe_dump(document)) + try: + service.start() + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(log["message"] == "script-file" and log["count"] == 1 for log in output) + finally: + service.stop() + + +@pytest.mark.parametrize("signal", ["profiles"]) +def test_unsupported_signals_fail_startup(tmp_path, signal): + service = make_service(tmp_path, "function process_log(log) end") + config = tmp_path / "typed.yaml" + document = yaml.safe_load(config.read_text()) + processors = document["pipeline"]["inputs"][0]["processors"] + processors[signal] = processors.pop("logs") + config.write_text(yaml.safe_dump(document)) + try: + with pytest.raises(FluentBitStartupError): + service.start() + assert "unknown processor 'profiles'" in log_text(service) + finally: + service.stop() + + +def test_unpublished_processor_name_is_not_registered(tmp_path): + service = make_service(tmp_path, "function process_log(log) end") + config = tmp_path / "typed.yaml" + document = yaml.safe_load(config.read_text()) + document["pipeline"]["inputs"][0]["processors"]["logs"][0]["name"] = "lua_typed" + config.write_text(yaml.safe_dump(document)) + try: + with pytest.raises(FluentBitStartupError): + service.start() + finally: + service.stop() + + +def test_record_edit_without_module_import(tmp_path): + service = make_service(tmp_path, """function process_log(log) + log.body:set("message", "no-import") + end""") + try: + service.start() + output = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(log["message"] == "no-import" for log in output) + finally: + service.stop() + + +@pytest.mark.parametrize("output_processor", [False, True]) +def test_trusted_jit_policy_runs(tmp_path, output_processor): + code = """local total=0 +for j=1,30 do for i=1,100000 do total=total+i end end +function process_log(log) + local sum=0; for i=1,50000 do sum=sum+i end + log.body:set('message', tostring(sum)) +end""" + service = make_service(tmp_path, code, output_processor=output_processor, + runtime_options={"jit": True, "instruction_limit": 0}) + try: + service.start() + rows = service.wait_for_condition(lambda: records(service), timeout=30) + assert all(row["message"] == "1250025000" for row in rows) + assert "edits rolled back" not in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("options", [{"jit": True}, {"jit": True, "instruction_limit": 1000}, + {"jit": False, "instruction_limit": 0}]) +def test_jit_and_instruction_budget_cannot_be_combined(tmp_path, options): + service = make_service(tmp_path, "function process_log(log) end", runtime_options=options) + try: + with pytest.raises(FluentBitStartupError): + service.start() + expected = ("jit requires instruction_limit=0" if options.get("jit") else + "instruction_limit must be between 1000 and 2147483647 with jit disabled") + assert expected in log_text(service) + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_audits.py b/tests/integration/scenarios/processor_fts/tests/test_fts_audits.py new file mode 100644 index 00000000000..95dc5017806 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_audits.py @@ -0,0 +1,97 @@ +"""Regressions for preservation, read-only context access and timestamp writes.""" +import json +from pathlib import Path + +import pytest +import yaml + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port +from utils.test_service import FluentBitTestService +from test_fts_policies import PolicyRunner, attributes, scalar + + +@pytest.fixture(scope="module", params=["input", "output"]) +def audit_runner(request, tmp_path_factory): + port = find_available_port() + source = {"name": "opentelemetry", "port": port} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + code = '''function process(log) + local action = log.body:get_string("text") + if action == "edit" then log.body:set("edited", true) + elseif action == "read" then + assert(log.resource.attributes:len() >= 0) + assert(log.scope.attributes:len() >= 0) + elseif action == "marker" then + log.timestamp.sec = 4294967295; log.timestamp.nsec = 1 + end + end''' + (source if request.param == "input" else output)["processors"] = { + "logs": [{"name": "fts", "code": code, "error_mode": "drop"}]} + config = tmp_path_factory.mktemp("fts-audits") / "audit.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + manager = FluentBitManager(str(config)) + try: + manager.start() + yield PolicyRunner(manager, port) + finally: + manager.stop() + + +@pytest.mark.parametrize("action", ["noop", "edit", "read"]) +def test_untouched_otel_fields_and_context_survive(audit_runner, action): + request = f"audit-{action}" + payload = audit_runner.payload("unused", request) + source = payload["resourceLogs"][0] + source["scopeLogs"][0]["scope"].pop("attributes") + logs = source["scopeLogs"][0]["logRecords"] + for index, record in enumerate(logs): + record["traceId"] = "0" * 32 + record["spanId"] = "0" * 16 + record["severityNumber"] = 30 + if index < 2: + record["body"]["kvlistValue"]["values"][0]["value"]["stringValue"] = action + audit_runner.send(payload) + resources = audit_runner.wait_for_condition(lambda: audit_runner.collect(request)) + assert len(resources) == 1 + assert len(resources[0]["scopeLogs"]) == 1 + exported = resources[0]["scopeLogs"][0]["logRecords"] + assert len(exported) == 3 + for index, record in enumerate(exported): + assert record["traceId"] == "0" * 32 and record["spanId"] == "0" * 16 + assert record["severityNumber"] == 30 + assert ("edited" in scalar(record["body"])) == (action == "edit" and index < 2) + assert "edits rolled back" not in Path(audit_runner.manager.log_file).read_text() + + +def test_reserved_timestamp_drops_only_changed_record(audit_runner): + request = "audit-marker" + payload = audit_runner.payload("unused", request) + payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0]["body"]["kvlistValue"]["values"][0]["value"]["stringValue"] = "marker" + audit_runner.send(payload) + resources = audit_runner.wait_for_condition(lambda: audit_runner.collect(request)) + records = [record for resource in resources for scope in resource["scopeLogs"] + for record in scope["logRecords"]] + assert [scalar(record["body"])["text"] for record in records] == ["operationB", "control"] + + +def test_plain_metadata_does_not_gain_otlp_fields(tmp_path): + config = tmp_path / "plain.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": { + "inputs": [{"name": "dummy", "dummy": json.dumps({"message": "plain"}), + "metadata": json.dumps({"trace_flags": 1, "event_name": "ordinary.metadata"}), "rate": 4}], + "outputs": [{"name": "stdout", "match": "*", "format": "otlp_json"}]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + records = [record for batch in batches for resource in batch["resourceLogs"] + for scope in resource["scopeLogs"] for record in scope["logRecords"]] + assert records + assert all("flags" not in record and "eventName" not in record for record in records) + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py b/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py new file mode 100644 index 00000000000..95c4553945c --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_native_interop.py @@ -0,0 +1,289 @@ +"""FTS extensions must preserve metric label processors and non-FTS exports.""" +import json +import os +import socket +from pathlib import Path + +import pytest +import requests +import yaml + +from utils.network import find_available_port +from utils.test_service import FluentBitTestService +from utils.fluent_bit_manager import FluentBitManager +from test_fts_signals import attr, metric, number_point +from google.protobuf.json_format import ParseDict +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from test_fts_policies import attributes + + +@pytest.mark.parametrize("fts", [False, True]) +@pytest.mark.parametrize("unlabeled", [False, True]) +def test_labels_and_static_labels_survive_otlp_export(tmp_path, fts, unlabeled): + source_port, destination_port = find_available_port(), find_available_port() + processors = ([{"name": "fts", "code": "function process(metric) end"}] if fts else []) + processors.append({"name": "labels", "insert": "environment production", "update": "region west", + "delete": "secret"}) + config = tmp_path / "labels.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": { + "inputs": [{"name": "opentelemetry", "port": source_port, "tag": "original", + "processors": {"metrics": processors}}, + {"name": "opentelemetry", "port": destination_port, "tag": "converted"}], + "outputs": [{"name": "opentelemetry", "match": "original", "host": "127.0.0.1", + "port": destination_port, "tls": "off", "add_label": "cluster test"}, + {"name": "stdout", "match": "converted", "format": "otlp_json"}]}})) + item = metric("gauge", "series") + points = [number_point(8), number_point(42)] + points[1]["startTimeUnixNano"] = "2147483649000000000" + points[1]["timeUnixNano"] = "2147483649123456789" + points[1]["flags"] = 3 + if unlabeled: + for point in points: + point["attributes"] = [] + item["gauge"]["dataPoints"] = points + payload = {"resourceMetrics": [{"scopeMetrics": [{"metrics": [item]}]}]} + service = FluentBitTestService(str(config)) + try: + service.start() + response = requests.post(f"http://127.0.0.1:{source_port}/v1/metrics", json=payload, timeout=10) + response.raise_for_status() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + exported = [p for b in batches for r in b.get("resourceMetrics", []) for s in r["scopeMetrics"] + for m in s["metrics"] for p in m["gauge"]["dataPoints"]] + # OTLP JSON input merges repeated series by labels, so the last point wins. + expected = [(42, "2147483649000000000", "2147483649123456789", 3)] + assert [(int(p["asInt"]), p["startTimeUnixNano"], p["timeUnixNano"], p["flags"]) + for p in exported] == expected + for point in exported: + values = attributes(point["attributes"]) + assert values["environment"] == "production" and values["cluster"] == "test" + assert "secret" not in values + if not unlabeled: + assert values["region"] == "west" + assert "cannot construct FTS" not in Path(service.flb.log_file).read_text() + finally: + service.stop() + + +@pytest.mark.parametrize("fts", [False, True]) +def test_metric_chunk_compatibility(tmp_path, fts): + if not os.environ.get("FTS_COMPAT_BINARY"): + # Always exercise an actual 5.1.0 chunk when that binary is unavailable. + test_5_1_0_histogram_fixture_survives_forward(tmp_path, fts) + return + forward_port, source_port = find_available_port(), find_available_port() + receiver_config = tmp_path / "receiver.yaml" + sender_config = tmp_path / "sender.yaml" + service = {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"} + receiver_config.write_text(yaml.safe_dump({"service": service, "pipeline": { + "inputs": [{"name": "forward", "listen": "127.0.0.1", "port": forward_port}], + "outputs": [{"name": "stdout", "match": "*"}]}})) + source = {"name": "opentelemetry", "port": source_port, "tag": "compat"} + if fts: + source["processors"] = {"metrics": [{"name": "fts", "code": "function process(metric) end"}]} + sender_config.write_text(yaml.safe_dump({"service": service, "pipeline": { + "inputs": [source], "outputs": [{"name": "forward", "match": "*", "host": "127.0.0.1", + "port": forward_port}]}})) + receiver = FluentBitManager(str(receiver_config), binary_path=os.environ["FTS_COMPAT_BINARY"]) + sender = FluentBitTestService(str(sender_config)) + items = [metric("gauge", "compat_gauge"), metric("sum", "compat_sum"), + metric("histogram", "compat_histogram")] + histogram = items[2]["histogram"]["dataPoints"][0] + histogram.update(bucketCounts=["5", "1", "3"], count="9", sum=12.0) + items[1]["sum"]["isMonotonic"] = False + value = {"stringValue": "nested"} + for _ in range(25): + value = {"kvlistValue": {"values": [{"key": "child", "value": value}]}} + items[0]["gauge"]["dataPoints"][0]["attributes"].append({"key": "deep", "value": value}) + payload = {"resourceMetrics": [{"scopeMetrics": [{"metrics": items}]}]} + try: + receiver.start() + sender.start() + message = ParseDict(payload, ExportMetricsServiceRequest()) + response = requests.post(f"http://127.0.0.1:{source_port}/v1/metrics", data=message.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, timeout=10) + response.raise_for_status() + text = sender.wait_for_condition(lambda: (text if all(name in text for name in ("compat_gauge", "compat_sum", "compat_histogram")) else None) + if (text := Path(receiver.log_file).read_text()) else None, timeout=30) + assert "cmt_decode_msgpack_create failed" not in text + assert " 8" in text or "=8" in text or "= 8" in text + histogram_line = next(line for line in text.splitlines() if "compat_histogram" in line) + # The internal wire format keeps cumulative buckets, as 5.1 expects. + assert "buckets = { 1=5, 2=6, +Inf=9 }, sum=12, count=9" in histogram_line + diagnostics = Path(sender.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + finally: + sender.stop() + receiver.stop() + + +@pytest.mark.parametrize("fts", [False, True]) +def test_5_1_0_histogram_fixture_survives_forward(tmp_path, fts): + port = find_available_port() + source = {"name": "forward", "listen": "127.0.0.1", "port": port} + if fts: + source["processors"] = {"metrics": [{"name": "fts", "code": "function process(metric) end"}]} + config = tmp_path / "legacy-histogram.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": [source], + "outputs": [{"name": "stdout", "match": "*", "format": "otlp_json"}]}})) + fixture = Path(__file__).resolve().parents[5] / "lib/cmetrics/tests/data/histogram-v5.1.0.msgpack" + assert fixture.is_file(), "the vendored 5.1.0 compatibility fixture is required" + data = fixture.read_bytes() + # PackedForward frame: [tag, binary chunk, {fluent_signal: metrics}]. + message = b"\x93\xa6legacy\xc6" + len(data).to_bytes(4, "big") + data + b"\x81\xadfluent_signal\x01" + service = FluentBitTestService(str(config)) + try: + service.start() + with socket.create_connection(("127.0.0.1", port), timeout=10) as connection: + connection.sendall(message) + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + rows = [m for b in batches for r in b.get("resourceMetrics", []) for s in r["scopeMetrics"] + for m in s["metrics"]] + assert len(rows) == 1 and rows[0]["name"] == "legacy_histogram" + point = rows[0]["histogram"]["dataPoints"][0] + assert [int(count) for count in point["bucketCounts"]] == [5, 1, 3] + assert int(point["count"]) == 9 + assert point["sum"] == 12.0 + diagnostics = Path(service.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("signal", ["metrics", "traces"]) +@pytest.mark.parametrize("export", ["stdout", "otlp"]) +def test_without_fts_json_anyvalue_preservation(tmp_path, signal, export): + from test_fts_signals import span + + source_port, destination_port = find_available_port(), find_available_port() + source = {"name": "opentelemetry", "port": source_port, "tag": "source"} + inputs = [source] + outputs = [{"name": "stdout", "match": "source", "format": "otlp_json"}] + if export == "otlp": + inputs.append({"name": "opentelemetry", "port": destination_port, "tag": "exported"}) + outputs = [{"name": "opentelemetry", "match": "source", "host": "127.0.0.1", + "port": destination_port}, + {"name": "stdout", "match": "exported", "format": "otlp_json"}] + config = tmp_path / "native-anyvalue.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": inputs, "outputs": outputs}})) + values = [attr("healthy", "kept"), + {"key": "negative", "value": {"intValue": "-9223372036854775808"}}, + {"key": "positive", "value": {"intValue": "9223372036854775807"}}, + {"key": "bytes", "value": {"bytesValue": "AAH/"}}, + {"key": "overflow", "value": {"intValue": "18446744073709551615"}}, + {"key": "invalid", "value": {"boolValue": "wrong"}}] + if signal == "metrics": + item = metric("gauge", "native-values") + item["gauge"]["dataPoints"][0]["attributes"] = values + resource_key, scope_key, item_key = "resourceMetrics", "scopeMetrics", "metrics" + else: + item = span("native-values") + item["attributes"] = values + resource_key, scope_key, item_key = "resourceSpans", "scopeSpans", "spans" + payload = {resource_key: [{scope_key: [{item_key: [item]}]}]} + service = FluentBitTestService(str(config)) + try: + service.start() + response = requests.post(f"http://127.0.0.1:{source_port}/v1/{signal}", json=payload, timeout=10) + response.raise_for_status() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + rows = [m for b in batches for r in b.get(resource_key, []) for s in r[scope_key] for m in s[item_key]] + assert len(rows) == 1 + entries = rows[0]["gauge"]["dataPoints"][0]["attributes"] if signal == "metrics" else rows[0]["attributes"] + fields = {entry["key"]: entry["value"] for entry in entries} + assert fields["healthy"] == {"stringValue": "kept"} + assert int(fields["negative"]["intValue"]) == -(2**63) + assert int(fields["positive"]["intValue"]) == 2**63-1 + assert fields["bytes"] == {"bytesValue": "AAH/"} + assert "overflow" not in fields and "invalid" not in fields + diagnostics = Path(service.flb.log_file).read_text() + assert "could not decode" not in diagnostics and "could not convert" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("export", ["stdout", "otlp"]) +def test_without_fts_unknown_signed_span_kind_keeps_siblings(tmp_path, export): + from test_fts_signals import span + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + import base64 + + source_port, destination_port = find_available_port(), find_available_port() + inputs = [{"name": "opentelemetry", "port": source_port, "tag": "source"}] + outputs = [{"name": "stdout", "match": "source", "format": "otlp_json"}] + if export == "otlp": + inputs.append({"name": "opentelemetry", "port": destination_port, "tag": "exported"}) + outputs = [{"name": "opentelemetry", "match": "source", "host": "127.0.0.1", "port": destination_port}, + {"name": "stdout", "match": "exported", "format": "otlp_json"}] + config = tmp_path / "signed-kind.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": inputs, "outputs": outputs}})) + items = [span("negative-kind"), span("healthy")] + items[0]["kind"] = -1 + for item in items: + for key in ("traceId", "spanId", "parentSpanId"): + item[key] = base64.b64encode(bytes.fromhex(item[key])).decode() + for link in item["links"]: + for key in ("traceId", "spanId"): + link[key] = base64.b64encode(bytes.fromhex(link[key])).decode() + request = ParseDict({"resourceSpans": [{"resource": {}, "scopeSpans": [{"scope": {}, "spans": items}]}]}, + ExportTraceServiceRequest()) + service = FluentBitTestService(str(config)) + try: + service.start() + response = requests.post(f"http://127.0.0.1:{source_port}/v1/traces", data=request.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, timeout=10) + response.raise_for_status() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + rows = [span for batch in batches for resource in batch.get("resourceSpans", []) + for scope in resource["scopeSpans"] for span in scope["spans"]] + # stored traces keep the span; an unknown kind is stored as unspecified + assert [(row["name"], row.get("kind", 0)) for row in rows] == [("negative-kind", 0), ("healthy", 2)] + diagnostics = Path(service.flb.log_file).read_text() + assert "could not decode" not in diagnostics and "could not convert" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +@pytest.mark.parametrize("fts", [False, True]) +def test_log_trace_flags_preserve_the_low_eight_bits(tmp_path, transport, fts): + from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + + port = find_available_port() + source = {"name": "opentelemetry", "port": port} + if fts: + source["processors"] = {"logs": [{"name": "fts", "code": "function process(log) end"}]} + config = tmp_path / "flags.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": [source], + "outputs": [{"name": "stdout", "match": "*", "format": "otlp_json"}]}})) + payload = {"resourceLogs": [{"scopeLogs": [{"logRecords": [{"flags": 300, + "timeUnixNano": "2147483648123456789", "body": {"kvlistValue": {"values": [attr("marker", "flags")]}}}]}]}]} + service = FluentBitTestService(str(config)) + try: + service.start() + if transport == "protobuf": + message = ParseDict(payload, ExportLogsServiceRequest()) + response = requests.post(f"http://127.0.0.1:{port}/v1/logs", data=message.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, timeout=10) + else: + response = requests.post(f"http://127.0.0.1:{port}/v1/logs", json=payload, timeout=10) + response.raise_for_status() + batches = service.wait_for_condition(lambda: [json.loads(line) for line in + Path(service.flb.log_file).read_text().splitlines() if line.startswith("{")], timeout=30) + rows = [log for batch in batches for resource in batch.get("resourceLogs", []) + for scope in resource["scopeLogs"] for log in scope["logRecords"]] + assert len(rows) == 1 and rows[0]["flags"] == 44 + diagnostics = Path(service.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py b/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py new file mode 100644 index 00000000000..a598ef69acc --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_pipeline.py @@ -0,0 +1,371 @@ +"""Real OTLP export, fan-out isolation, retry, and filesystem ingestion.""" +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +import requests +import yaml +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + +from utils.test_service import FluentBitTestService +from test_fts_policies import PolicyRunner, attributes, scalar + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("finish_native", [False, True]) +@pytest.mark.parametrize("native_neighbor", ["none", "before", "after"]) +def test_native_filter_chain_preserves_edits_and_groups(tmp_path, placement, finish_native, + native_neighbor): + script = tmp_path / "select.lua" + script.write_text('''function select_log(tag, timestamp, record) + if record.text ~= "operationA" then return -1, timestamp, record end + record.filter_ran = true + return 2, timestamp, record + end''') + processors = [{"name": "fts", "code": '''function process(log) + log.body:set("stage", "before") + log.resource.attributes:set("kept.text", log.body:get_string("text")) + end'''}, {"name": "lua", "script": str(script), "call": "select_log"}] + if native_neighbor != "none": + processors.insert(0 if native_neighbor == "before" else 1, + {"name": "content_modifier", "context": "body", "action": "insert", + "key": "native_ran", "value": "preserved"}) + if finish_native: + processors.append({"name": "fts", "code": '''function process(log) + assert(log.body:get_string("stage") == "before") + log.body:set("stage", "after") + end'''}) + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + (source if placement == "input" else output)["processors"] = {"logs": processors} + config = tmp_path / "mixed-chain.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + payload = PolicyRunner(None, 0).payload("unused", "mixed-chain") + logs = payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"] + logs.insert(2, json.loads(json.dumps(logs[0]))) + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/logs", + json=payload, timeout=15) + response.raise_for_status() + + def collect(): + rows = [] + for line in Path(service.flb.log_file).read_text().splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + batch = json.loads(line) + rows.extend((resource, log) for resource in batch["resourceLogs"] + for scope in resource["scopeLogs"] for log in scope["logRecords"]) + return rows if len(rows) >= 2 else None + + rows = service.wait_for_condition(collect, timeout=30) + assert len(rows) == 2 + for resource, log in rows: + body = scalar(log["body"]) + assert body["text"] == "operationA" + if native_neighbor != "none": + assert body["native_ran"] == "preserved" + assert body["filter_ran"] is True + assert body["stage"] == ("after" if finish_native else "before") + assert attributes(resource["resource"]["attributes"])["kept.text"] == "operationA" + assert all(scope["logRecords"] for scope in resource["scopeLogs"]) + diagnostics = Path(service.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + finally: + service.stop() + + +def test_group_end_markers_reach_native_processors(tmp_path): + """Native processors after FTS receive complete group end records.""" + processors = [{"name": "fts", "code": "function process(log) log.body:set('seen', true) end"}, + {"name": "sql", "query": "SELECT text, seen FROM STREAM;"}] + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", + "processors": {"logs": processors}} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + config = tmp_path / "fts-sql.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + payload = PolicyRunner(None, 0).payload("unused", "fts-sql") + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/logs", + json=payload, timeout=15) + response.raise_for_status() + + def collect(): + rows = [] + for line in Path(service.flb.log_file).read_text().splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + batch = json.loads(line) + rows.extend(log for resource in batch["resourceLogs"] + for scope in resource["scopeLogs"] for log in scope["logRecords"]) + return rows or None + + rows = service.wait_for_condition(collect, timeout=30) + assert rows and all(scalar(log["body"])["seen"] is True for log in rows) + finally: + service.stop() + +def protobuf_string_field(message, number): + # The suite's pinned OTel Python bindings predate event_name. Protobuf + # retains unknown fields, so verify field 12 on the exported wire itself. + data = message.SerializeToString() + position = 0 + result = [] + + def varint(): + nonlocal position + value, shift = 0, 0 + while True: + byte = data[position]; position += 1 + value |= (byte & 127) << shift + if byte < 128: return value + shift += 7 + assert shift < 70 + + while position < len(data): + key = varint(); wire = key & 7 + if wire == 0: varint() + elif wire == 1: position += 8 + elif wire == 5: position += 4 + else: + assert wire == 2 + length = varint() + if key >> 3 == number: + result.append(data[position:position + length].decode("utf-8")) + position += length + assert position <= len(data) + return result + + +@pytest.mark.parametrize("filesystem,replay", [(False, False), (True, False), (True, True)]) +def test_output_policy_isolated_across_routes_and_retries(tmp_path, filesystem, replay): + received, attempts = [], [] + blocked = {"value": replay} + + class Collector(BaseHTTPRequestHandler): + def do_POST(self): + message = ExportLogsServiceRequest.FromString(self.rfile.read(int(self.headers["Content-Length"]))) + attempts.append(message) + status = 503 if blocked["value"] or len(attempts) == 1 else 200 + if status == 200: + received.append(message) + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Collector) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + policy = '''function process(log) + local producer = log.body:get_string("text") + log.resource.attributes:set("service.name", producer) + log.scope.name = producer + log.body:set("export.policy", "sanitized") + log.body:remove("secret") + end''' + source = {"name":"opentelemetry", "port":"${FLUENT_BIT_TEST_LISTENER_PORT}"} + service_config = {"flush":0.2, "grace":1, "http_server":True, + "http_port":"${FLUENT_BIT_HTTP_MONITORING_PORT}", + "scheduler.base":1, "scheduler.cap":1} + if filesystem: + service_config["storage.path"] = str(tmp_path / "storage") + source["storage.type"] = "filesystem" + config = tmp_path / "pipeline.yaml" + config.write_text(yaml.safe_dump({"service":service_config, "pipeline":{ + "inputs":[source], "outputs":[ + {"name":"stdout", "match":"*", "format":"otlp_json"}, + {"name":"opentelemetry", "match":"*", "host":"127.0.0.1", "port":server.server_port, + "retry_limit":False, "processors":{"logs":[{"name":"fts", "code":policy}]}}, + ]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + payload = PolicyRunner(None, 0).payload("unused", "route-isolation") + payload["resourceLogs"][0]["resource"]["droppedAttributesCount"] = 7 + payload["resourceLogs"][0]["scopeLogs"][0]["scope"]["droppedAttributesCount"] = 5 + logs = payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"] + logs.insert(2, json.loads(json.dumps(logs[0]))) + for log in logs: + log["body"]["kvlistValue"]["values"].append( + {"key":"secret", "value":{"stringValue":"sensitive"}}) + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/logs",json=payload, timeout=15) + assert response.status_code in (200, 201) + if replay: + service.wait_for_condition(lambda: attempts, timeout=30) + service.stop() + original_log = Path(service.flb.log_file).read_text() + assert list((tmp_path / "storage").rglob("*.flb")), "filesystem chunk was not retained" + blocked["value"] = False + service = FluentBitTestService(str(config)) + service.start() + service.wait_for_condition(lambda: received, timeout=30) + service.stop() + if not replay: + original_log = Path(service.flb.log_file).read_text() + assert len(attempts) >= 2 + assert attempts[0].SerializeToString() == attempts[1].SerializeToString() + rows = [(resource, scope, log) for message in received for resource in message.resource_logs + for scope in resource.scope_logs for log in scope.log_records] + assert [dict((v.key, v.value.string_value) for v in log.body.kvlist_value.values)["text"] + for _, _, log in rows] == ["operationA", "operationB", "operationA", "control"] + for resource, scope, log in rows: + body = {v.key:v.value.string_value for v in log.body.kvlist_value.values} + assert body["export.policy"] == "sanitized" and "secret" not in body + assert {v.key:v.value.string_value for v in resource.resource.attributes}["service.name"] == body["text"] + assert scope.scope.name == body["text"] + assert resource.resource.dropped_attributes_count == 7 + assert scope.scope.dropped_attributes_count == 5 + assert log.dropped_attributes_count in (1, 2) + assert protobuf_string_field(log, 12) == ["application.event"] + assert log.time_unix_nano == 2147483648123456789 and log.flags in (1, 2) + raw_batches = [json.loads(line) for line in original_log.splitlines() + if line.startswith('{')] + raw_logs = [log for batch in raw_batches for resource in batch["resourceLogs"] + for scope in resource["scopeLogs"] for log in scope["logRecords"]] + assert len(raw_logs) == 4 + assert all(scalar(log["body"])["secret"] == "sensitive" for log in raw_logs) + assert all("export.policy" not in scalar(log["body"]) for log in raw_logs) + for batch in raw_batches: + for resource in batch["resourceLogs"]: + assert "service.name" not in attributes(resource["resource"]["attributes"]) + assert all(scope["scope"]["name"] == "scope" for scope in resource["scopeLogs"]) + finally: + service.stop() + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("rollback", [False, True]) +def test_pooled_batch_commit_rollback_and_retained_values(tmp_path, placement, rollback): + from test_fts_security_batches import BatchRunner + + code = ''' + local saved + function process(log) + if saved then + assert(saved:get_string("text") == "retained snapshot") + saved:set("text", "changed after commit") + end + log.body:set("text", "retained snapshot") + log.body:set("edited", true) + saved = log.body:copy() + if ROLLBACK and log.body:get_number("index") == 599 then + log.body:set("large", string.rep("x", 70000)) + end + end + '''.replace("ROLLBACK", "true" if rollback else "false") + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + (source if placement == "input" else output)["processors"] = {"logs": [ + {"name": "fts", "code": code, "error_mode": "keep", "output_limit": "65536"}]} + config = tmp_path / "pooled-batch.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + runner = BatchRunner(service.flb, service.flb_listener_port, "logs") + items = [{"timeUnixNano": "2147483648123456789", "body": {"kvlistValue": {"values": [ + {"key": "index", "value": {"intValue": str(index)}}, + {"key": "text", "value": {"stringValue": "original"}}]}}} for index in range(600)] + for _ in range(2): + _, rows = runner.send(items, "json") + assert len(rows) == 600 + bodies = [scalar(row["body"]) for row in rows] + assert [body["index"] for body in bodies] == list(range(600)) + assert all(body["text"] == ("original" if rollback else "retained snapshot") + for body in bodies) + assert all(("edited" not in body) if rollback else body["edited"] is True + for body in bodies) + diagnostics = runner.recent_log() + assert "policy failed" not in diagnostics + assert ("cannot construct policy output" in diagnostics) is rollback + finally: + service.stop() + + +@pytest.mark.parametrize("chain", ["direct", "native"]) +def test_output_windows_keep_otlp_resources_whole(tmp_path, chain): + """Large output chunks are processed in windows without splitting resources.""" + processors = [{"name": "fts", "code": '''function process(log) + log.body:set("seen", true) + log.resource.attributes:set("windowed", "yes") + end'''}] + if chain == "native": + processors.append({"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"}) + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + output = {"name": "stdout", "match": "*", "format": "otlp_json", + "processors": {"logs": processors}} + config = tmp_path / "windows.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 1, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + resources = 4 + per_resource = 1500 + service = FluentBitTestService(str(config)) + try: + service.start() + payload = {"resourceLogs": [{ + "resource": {"attributes": [{"key": "res.id", "value": {"stringValue": f"r{res}"}}]}, + "scopeLogs": [{"scope": {"name": "windows"}, "logRecords": [ + {"timeUnixNano": str(1700000000000000000 + index), + "body": {"kvlistValue": {"values": [ + {"key": "index", "value": {"intValue": str(index)}}, + {"key": "text", "value": {"stringValue": "w" * 48}}]}}} + for index in range(per_resource)]}]} for res in range(resources)]} + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/logs", + json=payload, timeout=30) + response.raise_for_status() + + def collect(): + seen = {} + batches = [] + for line in Path(service.flb.log_file).read_text().splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + batch = json.loads(line) + batches.append(batch) + for resource in batch["resourceLogs"]: + res_id = attributes(resource["resource"]["attributes"])["res.id"] + for scope in resource["scopeLogs"]: + seen.setdefault(res_id, []).extend(scope["logRecords"]) + total = sum(len(logs) for logs in seen.values()) + return (seen, batches) if total >= resources * per_resource else None + + seen, batches = service.wait_for_condition(collect, timeout=60) + assert sorted(seen) == [f"r{res}" for res in range(resources)] + for logs in seen.values(): + bodies = [scalar(log["body"]) for log in logs] + assert [body["index"] for body in bodies] == list(range(per_resource)) + assert all(body["seen"] is True and body["text"] == "w" * 48 for body in bodies) + assert all(body.get("following") == ("native" if chain == "native" else None) + for body in bodies) + for batch in batches: + ids = [attributes(resource["resource"]["attributes"])["res.id"] + for resource in batch["resourceLogs"]] + # a resource split across windows would be emitted more than once + assert len(ids) == len(set(ids)) + for resource in batch["resourceLogs"]: + assert attributes(resource["resource"]["attributes"])["windowed"] == "yes" + diagnostics = Path(service.flb.log_file).read_text() + assert "cannot construct" not in diagnostics and "policy failed" not in diagnostics + assert "invalid log window" not in diagnostics + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_policies.py b/tests/integration/scenarios/processor_fts/tests/test_fts_policies.py new file mode 100644 index 00000000000..8efb0342f24 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_policies.py @@ -0,0 +1,598 @@ +"""Log-policy matrix inspired by OTTL's log transform processor tests. + +These are independent end-to-end assertions, not an OTTL interpreter or a +claim of parity with the complete OTTL function catalog. Every policy runs +through both input and output processor placement and OTLP JSON export. +""" + +import copy +import json +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import pytest +import requests +import yaml + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port + + +# (name, record policy, inherited-context policy, expected target, values for A/B) +POLICIES = [ + ("set", 'attrs:set("test", "pass")', "", "attrs", ["pass", "pass"]), + ("condition_true", 'if body:get_string("text") == "operationA" then attrs:set("test", "pass") end', + "", "attrs", ["pass", None]), + ("condition_false", 'if body:get_string("text") == "absent" then attrs:set("test", "pass") end', + "", "attrs", [None, None]), + ("resource_condition", 'if event.resource.attributes:get_string("host.name") == "localhost" ' + 'then attrs:set("test", "pass") end', "", "attrs", ["pass", "pass"]), + ("scope_condition", 'if event.scope.name == "scope" ' + 'then attrs:set("test", "pass") end', "", "attrs", ["pass", "pass"]), + ("scope_wrong_version", 'if event.scope.version == "2" ' + 'then attrs:set("test", "pass") end', "", "attrs", [None, None]), + ("severity_text", 'if attrs:get_string("http.path") == "/health" then otlp:set("severity_text", "OK") end', + "", "severityText", ["OK", "OK"]), + ("severity_number", 'if otlp:get_number("severity_number") == 9 then ' + 'otlp:set("severity_number", fb.int64(10)) end', "", "severityNumber", [10, 10]), + ("flags_condition", 'if fb.log_field(event, "trace_flags"):number() == 1 then attrs:set("test", "pass") end', + "", "attrs", ["pass", None]), + ("dropped_count_condition", 'if otlp:get_number("dropped_attributes_count") == 1 then ' + 'attrs:set("test", "pass") end', "", "attrs", ["pass", None]), + ("trace_id_condition", 'if otlp:get("trace_id"):value() == string.char(1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16) ' + 'then attrs:set("test", "pass") end', "", "attrs", ["pass", "pass"]), + ("span_id_condition", 'if otlp:get("span_id"):value() == string.char(1,2,3,4,5,6,7,8) ' + 'then attrs:set("test", "pass") end', "", "attrs", ["pass", "pass"]), + ("pattern_condition", 'if body:get_string("text"):match("operation[AC]") then ' + 'attrs:set("test", "pass") end', "", "attrs", ["pass", None]), + ("replace_pattern", 'attrs:set("http.method", attrs:get_string("http.method"):gsub("g.t", "post"))', + "", "http.method", ["post", "post"]), + ("replace_all_values", 'for _, k in ipairs(attrs:keys()) do local v = attrs:get(k); ' + 'if v:type() == "string" then attrs:set(k, (v:value():gsub("get", "post"))) end end', + "", "http.method", ["post", "post"]), + ("rename_key", 'attrs:set("url", attrs:get("http.url")); attrs:remove("http.url")', + "", "rename", ["http://localhost/health", "http://localhost/health"]), + ("delete_key", 'if body:get_string("text") == "operationA" then attrs:remove("http.url") end', + "", "delete", [True, False]), + ("delete_matching_keys", 'for _, k in ipairs(attrs:keys()) do ' + 'if k:match("^http%..*t") then attrs:remove(k) end end', "", "delete_matching", [True, True]), + ("keep_keys", 'fb.keep_keys(attrs, {"http.method", "request.id"})', "", "keep", [True, True]), + ("concat", 'attrs:set("test", attrs:get_string("http.method") .. ": " .. attrs:get_string("http.url"))', + "", "attrs", ["get: http://localhost/health", "get: http://localhost/health"]), + ("split", 'local a = fb.array(); for s in attrs:get_string("flags"):gmatch("[^|]+") do ' + 'a:append(s) end; attrs:set("test", a)', "", "attrs", [["A", "B", "C"], ["C", "D"]]), + ("split_missing", 'local s = attrs:get_string("absent"); if s then ' + 'attrs:set("test", s:sub(1, 1)) end', "", "attrs", [None, None]), + ("substring", 'attrs:set("test", attrs:get_string("total.string"):sub(4, 6))', + "", "attrs", ["456", "678"]), + ("substring_missing", 'local s = attrs:get_string("absent"); if s then ' + 'attrs:set("test", s:sub(4, 6)) end', "", "attrs", [None, None]), + ("array_literal", 'local a = fb.array(); for _, s in ipairs({"A", "B", "C"}) do ' + 'a:append(s) end; attrs:set("test", a)', "", "attrs", [["A", "B", "C"], ["A", "B", "C"]]), + ("lower", 'attrs:set("test", body:get_string("text"):lower())', + "", "attrs", ["operationa", "operationb"]), + ("upper", 'attrs:set("test", body:get_string("text"):upper())', + "", "attrs", ["OPERATIONA", "OPERATIONB"]), + ("snake", 'attrs:set("test", (body:get_string("text"):gsub("(%l)(%u)", "%1_%2")):lower())', + "", "attrs", ["operation_a", "operation_b"]), + ("camel", 'local s = body:get_string("snake"); ' + 'attrs:set("test", (s:gsub("_(%l)", string.upper)))', + "", "attrs", ["operationA", "operationB"]), + ("merge_insert", 'local values = { ["http.method"] = "post", json_test = "pass" }; ' + 'for k, v in pairs(values) do if not attrs:get(k) then attrs:set(k, v) end end', + "", "merge_insert", ["pass", "pass"]), + ("merge_update", 'local values = { ["http.method"] = "post", absent = "ignored" }; ' + 'for k, v in pairs(values) do if attrs:get(k) then attrs:set(k, v) end end', + "", "http.method", ["post", "post"]), + ("merge_upsert", 'for k, v in pairs({ ["http.method"] = "post", new = "added" }) do ' + 'attrs:set(k, v) end', "", "merge_upsert", ["added", "added"]), + ("limit_zero", 'fb.keep_keys(attrs, {"request.id"})', "", "empty", [True, True]), + ("math_log", 'attrs:set("test", math.log(1))', "", "attrs", [0.0, 0.0]), + ("nested_body", 'body:get("metadata"):set("uid", "12345")', + "", "nested", ["12345", "12345"]), + ("record_cache", 'local cache = { text = body:get_string("text") }; attrs:set("test", cache.text)', + "", "attrs", ["operationA", "operationB"]), + ("resource_set", "", 'event.resource.attributes:set("test", "pass")', + "resource", "pass"), + ("resource_false", "", 'local a = event.resource.attributes; if a:get_string("host.name") == "wrong" ' + 'then a:set("test", "pass") end', "resource", None), + ("resource_schema", "", 'event.resource.schema_url = "https://schema.example/resource"', + "resource_schema", "https://schema.example/resource"), + ("scope_set", "", 'event.scope.attributes:set("test", "pass")', "scope", "pass"), + ("scope_schema", "", 'event.scope.schema_url = "https://schema.example/scope"', + "scope_schema", "https://schema.example/scope"), + ("scope_name", "", 'event.scope.name = "new-scope"', "scope_name", "new-scope"), + ("scope_version", "", 'event.scope.version = "2.0"', "scope_version", "2.0"), + ("timestamp", 'event.timestamp.sec = 2147483648; event.timestamp.nsec = 987654321', + "", "timeUnixNano", ["2147483648987654321", "2147483648987654321"]), + ("observed_timestamp", 'otlp:set("observed_timestamp", fb.uint64("2147483648987654321"))', + "", "observedTimeUnixNano", ["2147483648987654321", "2147483648987654321"]), + ("field_precedence", 'event.metadata:set("severity_text", "ROOT"); ' + 'otlp:set("severity_text", "NESTED"); ' + 'assert(fb.log_field(event, "severity_text"):value() == "NESTED")', + "", "severityText", ["NESTED", "NESTED"]), + ("field_zero", 'otlp:set("trace_flags", fb.uint64(0)); ' + 'event.metadata:set("trace_flags", fb.uint64(1)); ' + 'attrs:set("test", fb.log_field(event, "trace_flags"))', "", "attrs", [0, 0]), + ("field_missing", 'assert(fb.log_field(event, "absent") == nil)', + "", "attrs", [None, None]), + ("signed_min", 'attrs:set("test", fb.int64("-9223372036854775808"))', + "", "attrs", [-9223372036854775808, -9223372036854775808]), + ("signed_max", 'attrs:set("test", fb.int64("9223372036854775807"))', + "", "attrs", [9223372036854775807, 9223372036854775807]), + ("empty_array", 'attrs:set("test", fb.array())', "", "attrs", [[], []]), + ("empty_map", 'attrs:set("test", fb.map())', "", "attrs", [{}, {}]), + ("binary", 'attrs:set("test", fb.bytes("a\\0b"))', "", "bytes", ["YQBi", "YQBi"]), + ("nested_array", 'local a = fb.array(); local m = fb.map(); m:set("nested", true); ' + 'a:append(m); a:append(fb.int64(7)); attrs:set("test", a)', + "", "attrs", [[{"nested": True}, 7], [{"nested": True}, 7]]), + ("rollback", 'body:set("text", "partial"); attrs:set("test", "partial"); error("matrix-rollback")', + "", "attrs", [None, None]), + ("context_rollback", "", 'event.resource.attributes:set("test", "partial"); error("matrix-context-rollback")', + "resource", None), + ("drop_first", 'if body:get_string("text") == "operationA" then return fb.DROP end', + "", "drop", [True, False]), + ("drop_both", 'return fb.DROP', "", "drop", [True, True]), +] + + +# Operational recipes: producer enrichment stays isolated; request identity remains +# on the event. Parse/normalize helpers remove the cache/type plumbing from policies. +POLICIES.extend([ + ("producer_from_record", 'event.resource.attributes:set("service.name", body:get_string("text"))', + "", "producer", ["operationA", "operationB"]), + ("scope_from_record", 'event.scope.name = body:get_string("text")', + "", "scope_name_record", ["operationA", "operationB"]), + ("tenant_on_event", 'attrs:set("tenant.id", body:get_string("text"))', + "", "tenant", ["operationA", "operationB"]), + ("named_severity", 'fb.set_severity(event, "Warning")', + "", "named_severity", [13, 13]), + ("fractional_time", 'event.timestamp = fb.parse_time("2024-11-19T18:05:24.0300402+00:00")', + "", "timeUnixNano", ["1732039524030040200", "1732039524030040200"]), + ("correlation", 'fb.set_trace_context(event, "abcdef0123456789abcdef0123456789", "abcdef0123456789")', + "", "correlation", [True, True]), + ("json_normalize", 'local parsed = fb.parse_json(\'{"Level":"Information","Properties":{"customer":"acme"}}\'); ' + 'fb.set_severity(event, parsed:get_string("Level")); ' + 'attrs:set("customer", parsed:get("Properties"):get("customer"))', + "", "json_normalize", ["acme", "acme"]), + ("recursive_privacy", 'local parsed = fb.parse_json(\'{"Authorization":"bearer","nested":{"password":"secret"},' + '"headers":[{"AUTHORIZATION":"token"}]}\'); ' + 'fb.redact(parsed, {"authorization", "password"}); body:set("sanitized", parsed)', + "", "privacy", [True, True]), + ("flags_preserve", 'otlp:set("trace_flags", fb.uint64(255))', + "", "flags", [255, 255]), + ("event_name", 'otlp:set("event_name", "policy.event")', + "", "eventName", ["policy.event", "policy.event"]), + ("dropped_count", 'otlp:set("dropped_attributes_count", fb.uint64(0))', + "", "droppedAttributesCount", [0, 0]), +]) + +# Boundary/failure cases supplement the OTTL-inspired policy families. Each +# failure happens after a mutation, so the output must prove full rollback. +INVALID_OPERATIONS = { + "signed_overflow": 'fb.int64("9223372036854775808")', + "signed_underflow": 'fb.int64("-9223372036854775809")', + "unsigned_overflow": 'fb.uint64("18446744073709551616")', + "unsigned_negative": 'fb.uint64("-1")', + "integer_whitespace": 'fb.int64(" 1")', + "integer_plus": 'fb.int64("+1")', + "integer_empty": 'fb.int64("")', + "integer_nul": 'fb.int64("1\\0")', + "integer_fraction": 'fb.int64(1.5)', + "integer_unsafe_number": 'fb.uint64(9007199254740992)', + "integer_nan": 'fb.int64(0/0)', + "unsafe_number_read": 'fb.uint64("18446744073709551615"):number()', + "nonfinite_write": 'body:set("bad", math.huge)', + "nil_write": 'body:set("bad", nil)', + "table_write": 'body:set("bad", {})', + "map_numeric_key": 'body:get(1)', + "map_numeric_set_key": 'body:set(1, "bad")', + "scalar_get": 'body:get("text"):get("bad")', + "scalar_set": 'body:get("text"):set("bad", "bad")', + "scalar_len": 'body:get("text"):len()', + "scalar_keys": 'body:get("text"):keys()', + "get_string_wrong_type": 'body:get_string("metadata")', + "get_bool_wrong_type": 'body:get_bool("text")', + "get_number_wrong_type": 'body:get_number("text")', + "ensure_map_conflict": 'body:ensure_map("text")', + "bytes_nonstring": 'fb.bytes(3)', + "map_constructor_args": 'fb.map({})', + "array_constructor_args": 'fb.array({})', + "empty_array_index": 'fb.array():get(1)', + "array_index_zero": 'local a = fb.array(); a:append(true); a:get(0)', + "array_index_fraction": 'local a = fb.array(); a:append(true); a:get(1.5)', + "array_index_string": 'local a = fb.array(); a:append(true); a:get("1")', + "array_index_nan": 'local a = fb.array(); a:append(true); a:get(0/0)', + "array_index_infinity": 'local a = fb.array(); a:append(true); a:get(math.huge)', + "array_set_out_of_bounds": 'fb.array():set(1, true)', + "map_append": 'body:append(true)', + "body_root_array": 'event.body = fb.array()', + "metadata_root_array": 'event.metadata = fb.array()', + "timestamp_negative": 'event.timestamp.sec = -1', + "timestamp_overflow": 'event.timestamp.sec = 4294967296', + "timestamp_fraction": 'event.timestamp.sec = 1.5', + "timestamp_nan": 'event.timestamp.sec = 0/0', + "timestamp_nsec_negative": 'event.timestamp.nsec = -1', + "timestamp_nsec_overflow": 'event.timestamp.nsec = 1000000000', + "timestamp_nsec_fraction": 'event.timestamp.nsec = 0.5', + "timestamp_string": 'event.timestamp.sec = "1"', + "timestamp_wrong_shape": 'event.timestamp = {}', + "nonstring_error": 'error({message="typed error"})', + "invalid_return": 'do return "invalid" end', + "stale_view": 'local child = body:get("metadata"):get("uid"); ' + 'body:remove("metadata"); child:value()', +} +INVALID_OPERATIONS.update({ + "json_duplicate": 'fb.parse_json(\'{"secret":"one","secret":"two"}\')', + "json_trailing": 'fb.parse_json(\'{"ok":1} trailing\')', + "json_overflow": 'fb.parse_json("18446744073709551616")', + "json_invalid": 'fb.parse_json("{broken")', + "severity_unknown": 'fb.set_severity(event, "unexpected")', + "severity_double": 'otlp:set("severity_number", 9)', + "flags_bounds": 'otlp:set("trace_flags", fb.uint64(256))', + "severity_bounds": 'otlp:set("severity_number", fb.int64(25))', + "trace_bad_length": 'fb.set_trace_context(event, "abc", "abcdef0123456789")', + "trace_all_zero": 'fb.set_trace_context(event, string.rep("0",32), "abcdef0123456789")', + "trace_wrong_type": 'otlp:set("trace_id", "abcdef0123456789abcdef0123456789")', + "time_missing_zone": 'fb.parse_time("2024-11-19T18:05:24")', + "time_bad_leap": 'fb.parse_time("2023-02-29T12:00:00Z")', + "time_bad_fraction": 'fb.parse_time("2024-02-29T12:00:00.1234567890Z")', + "time_bad_zone": 'fb.parse_time("2024-02-29T12:00:00+24:00")', + "time_bad_calendar": 'fb.parse_time("2024-02-30T12:00:00Z")', + "scope_invalid_shape": 'event.context.attributes:set("scope", true)', + "scope_invalid_name": 'event.context.attributes:get("scope"):set("name", fb.int64(1))', + "resource_invalid_attrs": 'event.context.attributes:get("resource"):set("attributes", false)', + "raw_time_conflict": 'otlp:set("timestamp", fb.uint64(1))', + "zero_trace": 'otlp:set("trace_id", fb.bytes(string.rep(string.char(0),16)))', + "invalid_utf8": 'attrs:set("bad", string.char(255))', + "invalid_utf8_key": 'attrs:set(string.char(255), "bad")', + "overlong_utf8": 'attrs:set("bad", string.char(192,128))', + "surrogate_utf8": 'attrs:set("bad", string.char(237,160,128))', + "otel_unsigned_overflow": 'attrs:set("bad", fb.uint64("18446744073709551615"))', + "otel_nul_string": 'attrs:set("bad", "a" .. string.char(0) .. "b")', + "otel_nul_key": 'attrs:set("a" .. string.char(0) .. "b", "bad")', + "root_flags_invalid": 'otlp:remove("trace_flags"); event.metadata:set("trace_flags", "bad")', + "event_name_invalid": 'otlp:set("event_name", fb.int64(1))', + "context_metadata_gc": 'local replacement=fb.map(); for _,k in ipairs(event.context.metadata:keys()) do replacement:set(k,event.context.metadata:get(k)) end; event.context.metadata=replacement; collectgarbage()', + "context_replace": 'event.context = {}', + "context_root_replace": 'event.context.attributes = fb.map()', + "resource_alias_replace": 'event.resource = {}', + "transport_id_edit": 'event.context.metadata:set("resource_id", fb.int64(9))', +}) +POLICIES.extend(("bad_" + name, 'body:set("text", "partial"); attrs:set("test", "partial"); ' + code, + "", "attrs", [None, None]) for name, code in INVALID_OPERATIONS.items()) + + +def scalar(value): + if "stringValue" in value: + return value["stringValue"] + if "intValue" in value: + return int(value["intValue"]) + if "doubleValue" in value: + return value["doubleValue"] + if "boolValue" in value: + return value["boolValue"] + if "bytesValue" in value: + return value["bytesValue"] + if "arrayValue" in value: + return [scalar(item) for item in value["arrayValue"].get("values", [])] + if "kvlistValue" in value: + return attributes(value["kvlistValue"].get("values", [])) + return None + + +def attributes(items): + return {item["key"]: scalar(item["value"]) for item in items} + + +def any_value(value): + if isinstance(value, str): + return {"stringValue": value} + if isinstance(value, dict): + return {"kvlistValue": {"values": kvpairs(value)}} + raise AssertionError(f"unsupported fixture value: {value!r}") + + +def kvpairs(values): + return [{"key": key, "value": any_value(value)} for key, value in values.items()] + + +def policy_code(): + definitions = [] + for name, log_code, context_code, _, _ in POLICIES: + definitions.append(f'logs[{json.dumps(name)}] = function(event)\n' + 'local body = event.body; local attrs = event.attributes; ' + 'local otlp = event.metadata:get("otlp")\n' + log_code + '\n' + context_code + '\nend\n') + return 'local fb = require("fluentbit"); local logs = {}\n' + "\n".join(definitions) + ''' +function process_log(event) + if event.body:get_string("text") == "control" then return end + local name = event.resource.attributes:get_string("policy.case") + return logs[name](event) +end +''' + + +class PolicyRunner: + def __init__(self, manager, listener_port): + self.manager = manager + self.listener_port = listener_port + self.sequence = 0 + + def wait_for_condition(self, predicate, timeout=30): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = predicate() + if value: + return value + time.sleep(0.1) + raise AssertionError("timed out waiting for exported policy batch") + + def batches(self): + text = Path(self.manager.log_file).read_text() + return [json.loads(line) for line in text.splitlines(keepends=True) + if line.startswith("{") and line.endswith("\n")] + + def payload(self, name, request): + logs = [] + for i, text in enumerate(["operationA", "operationB", "control"]): + attrs = {"request.id": request, "http.method": "get", "http.path": "/health", + "http.url": "http://localhost/health", "flags": "A|B|C" if i == 0 else "C|D", + "total.string": "123456789" if i == 0 else "345678"} + body = {"text": text, "snake": "operation_a" if i == 0 else "operation_b", + "metadata": {"uid": "original"}} + logs.append({"timeUnixNano": "2147483648123456789", + "observedTimeUnixNano": "2147483648234567890", + "severityNumber": 9, "severityText": "INFO", "flags": 1 if i == 0 else 2, + + "eventName": "application.event", + "droppedAttributesCount": 1 if i == 0 else 2, + "traceId": "0102030405060708090a0b0c0d0e0f10", "spanId": "0102030405060708", + "attributes": kvpairs(attrs), "body": any_value(body)}) + return {"resourceLogs": [{ + "schemaUrl": "https://schema.example/original-resource", + "resource": {"attributes": kvpairs({"host.name": "localhost", "policy.case": name, + "policy.request": request})}, + "scopeLogs": [{"scope": {"name": "scope", "version": "1", "attributes": []}, + "schemaUrl": "https://schema.example/original-scope", "logRecords": logs}], + }]} + + def send(self, payload): + response = requests.post(f"http://127.0.0.1:{self.listener_port}/v1/logs", + json=payload, timeout=15) + assert response.status_code in (200, 201), response.text + + def collect(self, request): + found = [] + for batch in self.batches(): + for resource in batch["resourceLogs"]: + if attributes(resource["resource"]["attributes"]).get("policy.request") == request: + found.append(resource) + if any(scalar(log["body"]).get("text") == "control" + for resource in found for scope in resource["scopeLogs"] for log in scope["logRecords"]): + return found + return None + + def run(self, name): + self.sequence += 1 + request = f"{name}-{self.sequence}" + payload = self.payload(name, request) + log_offset = len(Path(self.manager.log_file).read_text()) + self.send(payload) + found = self.wait_for_condition(lambda: self.collect(request), timeout=30) + if not name.startswith("bad_") and name not in ("rollback", "context_rollback"): + diagnostics = Path(self.manager.log_file).read_text()[log_offset:] + assert "cannot construct" not in diagnostics + assert "policy failed" not in diagnostics + return payload["resourceLogs"][0], found + + +@pytest.fixture(scope="module", params=["input", "output"]) +def runner(request, tmp_path_factory): + config = tmp_path_factory.mktemp("typed-policies") / "policies.yaml" + processor = {"name": "fts", "code": policy_code(), "call": "process_log", + "error_mode": "keep"} + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + (source if request.param == "input" else output)["processors"] = {"logs": [processor]} + config.write_text(yaml.safe_dump({ + "service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}, + })) + # A module-owned manager survives the suite's per-test active-service + # cleanup. Fixture finalization still performs strict memory verification. + listener_port = find_available_port() + with pytest.MonkeyPatch.context() as env: + env.setenv("FLUENT_BIT_TEST_LISTENER_PORT", str(listener_port)) + env.setenv("FLUENT_BIT_HTTP_MONITORING_PORT", "0") + manager = FluentBitManager(str(config)) + try: + manager.start() + yield PolicyRunner(manager, listener_port) + finally: + manager.stop() + + +@pytest.mark.parametrize("policy", POLICIES, ids=[case[0] for case in POLICIES]) +def test_log_policy_matrix(runner, policy): + name, _, _, target, expected = policy + original, resources = runner.run(name) + if name.startswith("bad_"): + assert "edits rolled back" in Path(runner.manager.log_file).read_text() + pairs = [(resource, scope, log) for resource in resources for scope in resource["scopeLogs"] + for log in scope["logRecords"]] + logs = [log for _, _, log in pairs] + expected_count = 3 - sum(expected) if target == "drop" else 3 + assert len(logs) == expected_count + original_logs = original["scopeLogs"][0]["logRecords"] + kept_indices = [i for i in range(3) if target != "drop" or i == 2 or not expected[i]] + for i, (resource, scope, log) in zip(kept_indices, pairs): + resource_attrs = attributes(resource["resource"]["attributes"]) + scope_attrs = attributes(scope["scope"].get("attributes", [])) + if target == "resource": + assert resource_attrs.get("test") == (expected if i < 2 else None) + if target == "scope": + assert scope_attrs.get("test") == (expected if i < 2 else None) + if target == "resource_schema": + assert resource["schemaUrl"] == (expected if i < 2 else original["schemaUrl"]) + if target == "scope_schema": + assert scope["schemaUrl"] == (expected if i < 2 else original["scopeLogs"][0]["schemaUrl"]) + if target in ("scope_name", "scope_version"): + key = target.split("_")[1] + assert scope["scope"][key] == (expected if i < 2 else original["scopeLogs"][0]["scope"][key]) + original_log = original_logs[i] + if target == "producer": + assert resource_attrs.get("service.name") == (expected[i] if i < 2 else None) + if target == "scope_name_record": + assert scope["scope"]["name"] == (expected[i] if i < 2 else "scope") + baseline_attrs = attributes(original_log["attributes"]) + actual_attrs = attributes(log["attributes"]) + baseline_body = scalar(original_log["body"]) + actual_body = scalar(log["body"]) + wanted_attrs = copy.deepcopy(baseline_attrs) + wanted_body = copy.deepcopy(baseline_body) + if i < 2: + value = expected[i] if isinstance(expected, list) else expected + if target in ("attrs", "bytes") and value is not None: + wanted_attrs["test"] = value + elif target == "http.method": + wanted_attrs["http.method"] = value + elif target == "rename": + wanted_attrs["url"] = wanted_attrs.pop("http.url") + elif target == "delete" and value: + wanted_attrs.pop("http.url") + elif target == "delete_matching": + wanted_attrs.pop("http.method") + wanted_attrs.pop("http.path") + elif target == "keep": + wanted_attrs = {k: v for k, v in wanted_attrs.items() if k in ("http.method", "request.id")} + elif target == "empty": + wanted_attrs = {"request.id": wanted_attrs["request.id"]} + elif target == "merge_insert": + wanted_attrs["json_test"] = value + elif target == "merge_upsert": + wanted_attrs.update({"http.method": "post", "new": value}) + elif target == "nested": + wanted_body["metadata"]["uid"] = value + elif target == "tenant": + wanted_attrs["tenant.id"] = value + assert "tenant.id" not in resource_attrs + elif target == "json_normalize": + wanted_attrs["customer"] = value + elif target == "privacy": + wanted_body["sanitized"] = {"Authorization": "[REDACTED]", + "nested": {"password": "[REDACTED]"}, "headers": [{"AUTHORIZATION": "[REDACTED]"}]} + + assert actual_attrs == wanted_attrs + assert actual_body == wanted_body + for field in ("timeUnixNano", "observedTimeUnixNano", "severityText", "severityNumber", + "traceId", "spanId", "flags", "droppedAttributesCount", "eventName"): + wanted = expected[i] if target == field and i < 2 else original_log[field] + if name == "field_zero" and field == "flags" and i < 2: + wanted = 0 + if target == "flags" and field == "flags" and i < 2: + wanted = expected[i] + if target == "named_severity" and i < 2: + if field == "severityNumber": wanted = 13 + if field == "severityText": wanted = "Warning" + if target == "correlation" and i < 2: + if field == "traceId": wanted = "abcdef0123456789abcdef0123456789" + if field == "spanId": wanted = "abcdef0123456789" + if target == "json_normalize" and i < 2: + if field == "severityNumber": wanted = 9 + if field == "severityText": wanted = "Information" + assert log[field] == wanted, (name, field, log[field], wanted) + # The final control record proves that the batch was flushed, including + # policies that drop every selected record or fail their callback. + assert scalar(logs[-1]["body"])["text"] == "control" + + +def test_concurrent_requests_preserve_context(runner): + payloads = [runner.payload("resource_set", f"concurrent-{i}") for i in range(8)] + with ThreadPoolExecutor(max_workers=4) as executor: + list(executor.map(runner.send, payloads)) + for i in range(8): + resources = runner.wait_for_condition(lambda: runner.collect(f"concurrent-{i}"), timeout=30) + pairs = [(attributes(resource["resource"]["attributes"]), scalar(log["body"])) + for resource in resources for scope in resource["scopeLogs"] for log in scope["logRecords"]] + assert [body["text"] for _, body in pairs] == ["operationA", "operationB", "control"] + assert [attrs.get("test") for attrs, _ in pairs] == ["pass", "pass", None] + +@pytest.mark.parametrize("policy,field", [("producer_from_record", "resource"), ("scope_from_record", "scope")]) +def test_interleaved_producers_preserve_event_order(runner, policy, field): + request = f"interleaved-{policy}" + payload = runner.payload(policy, request) + logs = payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"] + logs.insert(2, copy.deepcopy(logs[0])) + runner.send(payload) + resources = runner.wait_for_condition(lambda: runner.collect(request)) + rows = [(resource, scope, scalar(log["body"])["text"]) + for resource in resources for scope in resource["scopeLogs"] for log in scope["logRecords"]] + assert [text for _, _, text in rows] == ["operationA", "operationB", "operationA", "control"] + for resource, scope, text in rows[:-1]: + if field == "resource": + assert attributes(resource["resource"]["attributes"])["service.name"] == text + else: + assert scope["scope"]["name"] == text + +@pytest.mark.parametrize("policy", ["producer_from_record", "scope_from_record", "named_severity", "correlation"]) +def test_protobuf_ingestion_preserves_context_and_exact_time(runner, policy): + from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + request_id = f"protobuf-{policy}" + payload = runner.payload(policy, request_id) + original = payload["resourceLogs"][0] + message = ExportLogsServiceRequest() + resource = message.resource_logs.add() + for key, value in attributes(original["resource"]["attributes"]).items(): + attr = resource.resource.attributes.add(); attr.key = key; attr.value.string_value = value + resource.schema_url = original["schemaUrl"] + scope = resource.scope_logs.add(); scope.scope.name = "scope"; scope.scope.version = "1" + scope.schema_url = original["scopeLogs"][0]["schemaUrl"] + for i, source in enumerate(original["scopeLogs"][0]["logRecords"]): + log = scope.log_records.add() + log.time_unix_nano = int(source["timeUnixNano"]) + log.observed_time_unix_nano = int(source["observedTimeUnixNano"]) + log.severity_number = 9; log.severity_text = "INFO"; log.flags = 257 + log.dropped_attributes_count = i + 1 + log.trace_id = bytes.fromhex(source["traceId"]); log.span_id = bytes.fromhex(source["spanId"]) + for key, value in scalar(source["body"]).items(): + item = log.body.kvlist_value.values.add(); item.key = key + if isinstance(value, str): item.value.string_value = value + else: + child = item.value.kvlist_value.values.add(); child.key = "uid"; child.value.string_value = "original" + for key, value in attributes(source["attributes"]).items(): + item = log.attributes.add(); item.key = key; item.value.string_value = value + response = requests.post(f"http://127.0.0.1:{runner.listener_port}/v1/logs", + data=message.SerializeToString(), headers={"Content-Type":"application/x-protobuf"}, + timeout=15) + assert response.status_code in (200, 201), response.text + resources = runner.wait_for_condition(lambda: runner.collect(request_id)) + pairs = [(resource, scope, log) for resource in resources for scope in resource["scopeLogs"] + for log in scope["logRecords"]] + assert [scalar(log["body"])["text"] for _, _, log in pairs] == ["operationA", "operationB", "control"] + for i, (resource, scope, log) in enumerate(pairs): + assert log["timeUnixNano"] == "2147483648123456789" + assert log["observedTimeUnixNano"] == "2147483648234567890" + assert log["flags"] == 1 and log["droppedAttributesCount"] == i + 1 + if policy == "producer_from_record": + assert attributes(resource["resource"]["attributes"]).get("service.name") == ( + ["operationA", "operationB"][i] if i < 2 else None) + if policy == "scope_from_record": + assert scope["scope"]["name"] == ["operationA", "operationB", "scope"][i] + assert log["severityNumber"] == (13 if policy == "named_severity" and i < 2 else 9) + assert log["traceId"] == ("abcdef0123456789abcdef0123456789" if policy == "correlation" and i < 2 + else "0102030405060708090a0b0c0d0e0f10") + + +def test_adjacent_scopes_share_resource_without_reordering(runner): + request_id = "adjacent-scopes" + payload = runner.payload("set", request_id) + scopes = payload["resourceLogs"][0]["scopeLogs"] + for name in ["second", "third"]: + scope = copy.deepcopy(scopes[0]); scope["scope"]["name"] = name; scopes.append(scope) + runner.send(payload) + resources = runner.wait_for_condition(lambda: runner.collect(request_id)) + assert len(resources) == 1 + assert [scope["scope"]["name"] for scope in resources[0]["scopeLogs"]] == ["scope", "second", "third"] + assert all(len(scope["logRecords"]) == 3 for scope in resources[0]["scopeLogs"]) diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_records.py b/tests/integration/scenarios/processor_fts/tests/test_fts_records.py new file mode 100644 index 00000000000..5e03623b347 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_records.py @@ -0,0 +1,620 @@ +"""Record commits preserve policy lifetimes, rollback and chain boundaries.""" +import json +import socket +import time + +from pathlib import Path + +import pytest +import yaml + +from utils.test_service import FluentBitTestService + + +def start_service(tmp_path, code, placement, *, following=None, **options): + processors = [{"name": "fts", "code": code, **options}] + if following: + processors.append(following) + source = {"name": "forward", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", "tag": "stream"} + output = {"name": "stdout", "match": "*", "format": "json_lines"} + (source if placement == "input" else output)["processors"] = {"logs": processors} + config = tmp_path / "records.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + service.start() + return service + + +def log_text(service): + return Path(service.flb.log_file).read_text() + + +def rows(service): + return [json.loads(line) for line in log_text(service).splitlines(keepends=True) + if line.startswith("{") and line.endswith("\n")] + + +class Raw(bytes): + """Preencoded MessagePack value.""" + + +def event_time(seconds): + return Raw(b"\xd7\x00" + seconds.to_bytes(4, "big") + bytes(4)) + + +def pack(value): + """Encode fixture values without an optional MessagePack dependency.""" + if isinstance(value, Raw): + return bytes(value) + if isinstance(value, int): + return b"\xcf" + value.to_bytes(8, "big") + if isinstance(value, str): + data = value.encode("utf-8") + return b"\xdb" + len(data).to_bytes(4, "big") + data + if isinstance(value, bytes): + return b"\xc6" + len(value).to_bytes(4, "big") + value + if isinstance(value, list): + return b"\xdd" + len(value).to_bytes(4, "big") + b"".join(pack(item) for item in value) + if isinstance(value, dict): + return b"\xdf" + len(value).to_bytes(4, "big") + b"".join( + pack(key) + pack(item) for key, item in value.items()) + raise TypeError(type(value)) + + +def send(service, records): + if isinstance(records, dict): + records = [records] + packed = b"".join(pack([1700000000, record]) for record in records) + message = pack(["stream", packed, {"size": len(records)}]) + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as connection: + connection.sendall(message) + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("chain", ["direct", "filter", "native", "replacement"]) +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +def test_records_commits_and_retained_views(tmp_path, placement, chain, error_mode): + code = ''' + local fb = require("fluentbit") + local previous, stale + function process(log) + if stale then + -- Record handles expire with their callback; copy() keeps values. + assert(not pcall(function() return stale.body:get_string("text") end)) + assert(not pcall(function() stale.metadata:set("late", true) end)) + end + if previous then + assert(previous.body:get_string("text") == "committed") + previous.body:set("text", "late mutation") + previous.body:get("values"):set(1, "late mutation") + previous.metadata:set("late", true) + previous.timestamp.sec = 1 + end + assert(log.timestamp.sec ~= 1) + assert(log.metadata:get("late") == nil) + local index = log.body:get_number("index") + if REPLACE then log.body = log.body:get("nested") end + log.body:set("text", "committed") + log.body:set("exact", fb.uint64("18446744073709551615")) + log.body:set("bytes", fb.bytes("a\\0b")) + log.body:set("empty", fb.array()) + fb.redact(log.body, {"password"}) + previous = {body = log.body:copy(), metadata = log.metadata:copy(), + timestamp = log.timestamp} + stale = log + if index == 2 then error("expected item failure") end + if index == 4 then return fb.DROP end + end + '''.replace("REPLACE", "true" if chain == "replacement" else "false") + following = None + if chain == "filter": + following = {"name": "grep", "regex": "text ^(original|committed)$"} + elif chain == "native": + following = {"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"} + service = start_service(tmp_path, code, placement, following=following, error_mode=error_mode) + try: + batch = [] + for index in range(10): + body = {"index": index, "text": "original", "password": "secret", "values": ["original"]} + batch.append({**body, "nested": dict(body)}) + send(service, batch) + expected = [index for index in range(10) if index != 4 and + (error_mode == "keep" or index != 2)] + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= len(expected) + else None, timeout=45) + assert [row["index"] for row in output] == expected + for row in output: + failed = row["index"] == 2 + assert row["text"] == ("original" if failed else "committed") + assert row["values"] == ["original"] + assert row["date"] == 1700000000.0 + assert row["password"] == ("secret" if failed else "[REDACTED]") + if not failed: + assert row["exact"] == 18446744073709551615 + assert row["bytes"] == "a\0b" + assert row["empty"] == [] + if chain == "native": + assert row["following"] == "native" + diagnostics = log_text(service) + assert "cannot construct" not in diagnostics + assert diagnostics.count("log policy failed:") == 1 + assert "expected item failure" in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +def test_large_batch_edits_and_rollbacks_stay_record_local(tmp_path, placement, error_mode): + """Edits and rollbacks in a batch whose decoded records exceed the chunk value pool.""" + code = ''' + local fb = require("fluentbit") + function process(log) + local index = log.body:get_number("index") + log.body:remove("password") + log.body:get("nested"):set("state", "edited") + log.body:get("values"):append(index) + log.body:set("note", string.rep("n", index % 50 == 0 and 20000 or 8)) + if index % 97 == 5 then error("expected item failure") end + end + ''' + service = start_service(tmp_path, code, placement, error_mode=error_mode) + try: + count = 3000 + batch = [{"index": index, "password": "secret", "text": "x" * 64, + "nested": {"state": "original", "keep": index}, "values": ["first"]} + for index in range(count)] + send(service, batch) + expected = [index for index in range(count) + if error_mode == "keep" or index % 97 != 5] + output = service.wait_for_condition( + lambda: rows(service) if len(rows(service)) >= len(expected) else None, timeout=90) + assert [row["index"] for row in output] == expected + for row in output: + index = row["index"] + assert row["text"] == "x" * 64 + assert row["nested"]["keep"] == index + if index % 97 == 5: + assert row["password"] == "secret" + assert row["nested"]["state"] == "original" + assert row["values"] == ["first"] + assert "note" not in row + else: + assert "password" not in row + assert row["nested"]["state"] == "edited" + assert row["values"] == ["first", index] + assert len(row["note"]) == (20000 if index % 50 == 0 else 8) + assert "cannot construct" not in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +@pytest.mark.parametrize("chain", ["direct", "native"]) +def test_records_output_limit_rolls_back_complete_batch(tmp_path, placement, error_mode, chain): + # The native chain edits chunk values in place; failure must restore every record, + # including removed trailing keys, before the following processor runs. + code = '''local saved; function process(log) + if saved then saved:set("edited", false) end + saved = log.body:copy() + log.body:remove("text") + log.body:set("edited", true) + if log.body:get_number("index") == 599 then + log.body:set("large", string.rep("x", 70000)) + end + end''' + following = None + if chain == "native": + following = {"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"} + service = start_service(tmp_path, code, placement, following=following, + error_mode=error_mode, output_limit="65536") + try: + send(service, [{"index": index, "text": "original"} for index in range(600)]) + service.wait_for_condition(lambda: "cannot construct policy output" in log_text(service), + timeout=45) + if error_mode == "keep": + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) == 600 + else None, timeout=45) + assert [row["index"] for row in output] == list(range(600)) + assert all("edited" not in row and "large" not in row for row in output) + assert all(row["text"] == "original" for row in output) + assert all(row.get("following") == ("native" if chain == "native" else None) + for row in output) + # Stop flushes all pending output, so drop assertions cannot pass early. + service.stop() + if error_mode == "drop": + assert rows(service) == [] + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +def test_records_edit_fits_single_tree_budget(tmp_path, placement): + service = start_service(tmp_path, 'function process(log) log.body:set("edited", true) end', + placement, memory_limit="262144", error_mode="keep") + try: + send(service, {"payload": "x" * 120000}) + output = service.wait_for_condition(lambda: rows(service), timeout=45) + assert len(output) == 1 + assert output[0]["edited"] is True + assert output[0]["payload"] == "x" * 120000 + diagnostics = log_text(service) + assert "policy failed" not in diagnostics and "cannot construct" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +def test_records_depth_error_is_item_local(tmp_path, placement, error_mode): + code = '''local fb = require("fluentbit") + function process(log) + fb.redact(log.body, {"password"}) + log.body:set("edited", true) + if log.body:get_number("index") == 1 then + local nested = fb.map() + for i = 1,63 do + local parent = fb.map(); parent:set("next", nested); nested = parent + end + log.body:set("deep", nested) + end + end''' + service = start_service(tmp_path, code, placement, error_mode=error_mode) + try: + send(service, [{"index": index, "password": "secret"} for index in range(3)]) + expected = [0, 1, 2] if error_mode == "keep" else [0, 2] + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= len(expected) + else None, timeout=45) + assert [row["index"] for row in output] == expected + for row in output: + assert "deep" not in row + assert row["password"] == ("secret" if row["index"] == 1 else "[REDACTED]") + assert ("edited" in row) == (row["index"] != 1) + diagnostics = log_text(service) + assert "cannot construct policy output" not in diagnostics + assert diagnostics.count("log policy failed:") == 1 + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +def test_failed_records_do_not_poison_later_redactions(tmp_path, placement): + code = '''local fb = require("fluentbit") + function process(log) + log.body:set("temporary", string.rep("x", 4096)) + if log.body:get_number("index") % 100 ~= 99 then error("expected rejection") end + log.body:remove("temporary") + fb.redact(log.body, {"password"}) + end''' + service = start_service(tmp_path, code, placement, error_mode="drop", memory_limit="262144") + try: + send(service, [{"index": index, "password": "secret"} for index in range(2000)]) + # Wait for the complete output and its diagnostics together: slow + # memory-checker runs can emit them in separate flushes. + def ready(): + output = rows(service) + if len(output) >= 20 and "expected rejection" in log_text(service): + return output + return None + + output = service.wait_for_condition(ready, timeout=120) + assert [row["index"] for row in output] == list(range(99, 2000, 100)) + assert all(row["password"] == "[REDACTED]" and "temporary" not in row for row in output) + diagnostics = log_text(service) + assert "cannot construct" not in diagnostics + assert "memory limit" not in diagnostics + assert "expected rejection" in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +def test_empty_body_and_metadata_fork_on_first_write(tmp_path, placement): + code = '''local previous + function process(log) + if previous then previous:ensure_map("nested"):set("value", "late") end + assert(log.body:len() == 0 and log.metadata:len() == 0) + log.body:ensure_map("nested"):set("value", "body") + assert(log.metadata:len() == 0) + log.metadata:ensure_map("nested"):set("value", "metadata") + assert(log.body:get("nested"):get_string("value") == "body") + previous = log.body:copy() + end''' + service = start_service(tmp_path, code, placement) + try: + send(service, [{}, {}]) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 2 + else None, timeout=45) + assert [row["nested"] for row in output] == [{"value": "body"}, {"value": "body"}] + diagnostics = log_text(service) + assert "policy failed" not in diagnostics and "cannot construct" not in diagnostics + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("chain", ["direct", "native"]) +def test_view_cache_follows_writes(tmp_path, placement, chain): + # Views cache their resolved node; every write must invalidate that cache. + code = r'''local fb = require("fluentbit") + local stale + function process(log) + if stale then assert(not pcall(function() return stale:get_string("p") end)) end + local b = log.body + local h = b:get("h") + assert(h:get_string("p") == "x") + b:get("h"):set("p", "y") + assert(h:get_string("p") == "y") + b:remove("h") + assert(not pcall(function() return h:get_string("p") end)) + b:set("h", fb.map()) + b:get("h"):set("p", "z") + assert(h:get_string("p") == "z") + local first = b:get("values"):get(1) + b:get("values"):set(1, "b") + assert(first:value() == "b") + local copy = b:copy() + local copied = copy:get("h") + b:get("h"):set("p", "w") + assert(copied:get_string("p") == "z") + stale = b:get("h") + end''' + following = None + if chain == "native": + following = {"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"} + service = start_service(tmp_path, code, placement, following=following) + try: + send(service, [{"index": index, "h": {"p": "x"}, "values": ["a"]} for index in range(3)]) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 3 + else None, timeout=45) + assert [row["h"] for row in output] == [{"p": "w"}] * 3 + assert [row["values"] for row in output] == [["b"]] * 3 + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("chain", ["direct", "native"]) +def test_untouched_bodies_keep_original_values(tmp_path, placement, chain): + # Plain-log bodies are built only when a policy uses them; untouched bodies + # must leave the pipeline exactly as they arrived. + code = r'''local fb = require("fluentbit") + function process(log) + local index = log.timestamp.sec % 10 + if index == 1 then log.timestamp.sec = log.timestamp.sec + 100 end + if index == 2 then log.metadata:set("seen", true) end + if index == 3 then return fb.DROP end + if index == 4 then assert(log.body:get_string("text") == "original") end + if index == 5 then log.body = log.body:get("nested") end + end''' + following = None + if chain == "native": + following = {"name": "content_modifier", "context": "body", "action": "insert", + "key": "following", "value": "native"} + service = start_service(tmp_path, code, placement, following=following) + try: + records = [{"text": "original", "big": 18446744073709551615, "bytes": b"a\0b", + "nested": {"inner": "value", "list": [1, "two"]}} for _ in range(6)] + packed = b"".join(pack([1700000000 + index, record]) for index, record in enumerate(records)) + message = pack(["stream", packed, {"size": len(records)}]) + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as conn: + conn.sendall(message) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 5 + else None, timeout=45) + extra = {"following": "native"} if chain == "native" else {} + full = {"text": "original", "big": 18446744073709551615, "bytes": "a\0b", + "nested": {"inner": "value", "list": [1, "two"]}, **extra} + assert [row["date"] for row in output] == [1700000000.0, 1700000101.0, 1700000002.0, + 1700000004.0, 1700000005.0] + for position, row in enumerate(output[:4]): + row.pop("date") + # stdout renders record metadata under __internal__. + metadata = row.pop("__internal__", {}).get("log_metadata") + assert metadata == ({"seen": True} if position == 2 else None) + assert row == full + nested = output[4] + nested.pop("date") + assert nested == {"inner": "value", "list": [1, "two"], **extra} + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +NESTED_RECORD = {"top": "t", "n": {"a": {"b": "deep", "list": [{"x": 1}, [2, "three"]]}, "pw": "secret"}, + "arr": [{"pw": "s1"}, {"k": "v"}]} +NESTED_CASES = { + "read": ('assert(log.body:get("n"):get("a"):get_string("b") == "deep")', NESTED_RECORD), + "write": ('log.body:get("n"):get("a"):set("b", "changed")', + {**NESTED_RECORD, "n": {"a": {"b": "changed", "list": [{"x": 1}, [2, "three"]]}, + "pw": "secret"}}), + "redact": ('fb.redact(log.body, {"pw"})', + {**NESTED_RECORD, "n": {**NESTED_RECORD["n"], "pw": "[REDACTED]"}, + "arr": [{"pw": "[REDACTED]"}, {"k": "v"}]}), + "append": ('log.body:get("n"):get("a"):get("list"):append("x")', + {**NESTED_RECORD, "n": {"a": {"b": "deep", "list": [{"x": 1}, [2, "three"], "x"]}, + "pw": "secret"}}), + "replace": ('log.body:set("n", "flat")', {**NESTED_RECORD, "n": "flat"}), + "copy": ('log.body:set("dup", log.body:get("n"))', {**NESTED_RECORD, "dup": NESTED_RECORD["n"]}), +} + + +@pytest.mark.parametrize("placement", ["input", "output"]) +@pytest.mark.parametrize("case", list(NESTED_CASES)) +def test_nested_values_open_on_demand(tmp_path, placement, case): + # Nested plain-log containers are built only when a policy opens them. + statement, expected = NESTED_CASES[case] + code = 'local fb = require("fluentbit")\nfunction process(log) ' + statement + ' end' + service = start_service(tmp_path, code, placement) + try: + send(service, [NESTED_RECORD, NESTED_RECORD]) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 2 + else None, timeout=45) + for row in output: + row.pop("date") + assert row == expected + assert "policy failed" not in log_text(service) + finally: + service.stop() + + +def test_failed_chunk_encoding_does_not_leak_records(tmp_path): + """A chunk that cannot be encoded must not reach the next chunk's tag.""" + processors = [{"name": "fts", "code": "function process(log) end"}, + {"name": "content_modifier", "action": "insert", "key": "added", "value": "v"}] + source = {"name": "forward", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", + "processors": {"logs": processors}} + output = {"name": "stdout", "match": "tagtwo", "format": "json_lines"} + config = tmp_path / "encode.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + service.start() + try: + # The second record carries a millisecond timestamp outside the event time range. + first = pack([1700000000, {"msg": "one"}]) + pack([1700000000123, {"msg": "bad"}]) + second = pack([1700000001, {"msg": "two"}]) + for message in (pack(["tagone", first, {"size": 2}]), pack(["tagtwo", second, {"size": 1}])): + with socket.create_connection(("127.0.0.1", service.flb_listener_port), + timeout=30) as connection: + connection.sendall(message) + time.sleep(0.5) + service.wait_for_condition(lambda: rows(service), timeout=20, + description="tagtwo output") + time.sleep(1) + assert [row["msg"] for row in rows(service)] == ["two"] + finally: + service.stop() + + +@pytest.mark.parametrize("error_mode", ["keep", "drop"]) +def test_missing_transport_ids_are_group_local(tmp_path, error_mode): + """A group without transport IDs does not skip the policy for other groups.""" + service = start_service(tmp_path, "function process(log) log.body:set('edited', true) end", + "input", following={"name": "content_modifier", "action": "insert", + "key": "native", "value": "v"}, + error_mode=error_mode) + try: + start, end = event_time(0xFFFFFFFF), event_time(0xFFFFFFFE) + now = event_time(1700000000) + entries = [[[start, {"schema": "otlp", "resource_id": 0, "scope_id": 0}], + {"resource": {}, "scope": {"name": "good"}}], + [[now, {"otlp": {}}], {"msg": "healthy-otel"}], + [[end, {}], {}], + [[start, {"schema": "otlp"}], {"resource": {}, "scope": {"name": "bad"}}], + [[now, {"otlp": {}}], {"msg": "missing-ids"}], + [[end, {}], {}], + [[now, {}], {"msg": "healthy-plain"}]] + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as connection: + connection.sendall(pack(["stream", entries])) + output = service.wait_for_condition(lambda: rows(service) if len(rows(service)) >= 3 else None, + timeout=20, description="three records") + assert sorted(row["msg"] for row in output) == ["healthy-otel", "healthy-plain", "missing-ids"] + assert all(row["edited"] is True and row["native"] == "v" for row in output) + assert "cannot construct policy output" not in log_text(service) + finally: + service.stop() + + +def start_otlp_json_service(tmp_path, processors): + source = {"name": "forward", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", "tag": "stream"} + if processors: + source["processors"] = {"logs": processors} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + config = tmp_path / "otlp-json.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 1, "grace": 1, + "http_server": True, "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + service = FluentBitTestService(str(config)) + service.start() + return service + + +def otlp_resources(service): + """(resource attributes, record bodies) for every exported resource.""" + resources = [] + for line in log_text(service).splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + for resource in json.loads(line)["resourceLogs"]: + attrs = [(item["key"], item["value"]) for item in + resource.get("resource", {}).get("attributes", [])] + bodies = [{item["key"]: item["value"] for item in log["body"]["kvlistValue"]["values"]} + for scope in resource["scopeLogs"] for log in scope["logRecords"]] + resources.append((attrs, bodies)) + return resources + + +def raw_map(pairs): + """Map with explicit, possibly duplicated, keys.""" + return Raw(b"\xdf" + len(pairs).to_bytes(4, "big") + + b"".join(pack(key) + pack(value) for key, value in pairs)) + + +@pytest.mark.parametrize("chain", ["fts", "content_modifier", "fts_and_content_modifier"]) +def test_groups_split_across_appends_keep_their_records(tmp_path, chain): + """A group whose end arrives in a later append keeps every record.""" + fts = {"name": "fts", "code": "function process(log) log.body:set('edited', true) end"} + modifier = {"name": "content_modifier", "action": "insert", "key": "native", "value": "v"} + processors = {"fts": [fts], "content_modifier": [modifier], + "fts_and_content_modifier": [fts, modifier]}[chain] + service = start_otlp_json_service(tmp_path, processors) + try: + start, end = event_time(0xFFFFFFFF), event_time(0xFFFFFFFE) + now = event_time(1700000000) + head = [[[start, {"schema": "otlp", "resource_id": 0, "scope_id": 0}], + {"resource": {"attributes": {"service.name": "split"}}, "scope": {"name": "s"}}], + [[now, {"otlp": {}}], {"msg": "r1"}]] + tail = [[[now, {"otlp": {}}], {"msg": "r2"}], [[end, {}], {}]] + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as connection: + connection.sendall(pack(["stream", head])) + time.sleep(0.1) + connection.sendall(pack(["stream", tail])) + resources = service.wait_for_condition( + lambda: otlp_resources(service) if sum(len(b) for _, b in otlp_resources(service)) >= 2 + else None, timeout=20, description="two exported records") + # both records stay in the resource opened by the first append + assert len(resources) == 1 + attrs, bodies = resources[0] + assert attrs == [("service.name", {"stringValue": "split"})] + assert [body["msg"]["stringValue"] for body in bodies] == ["r1", "r2"] + for body in bodies: + if "fts" in chain: + assert body["edited"] == {"boolValue": True} + if "content_modifier" in chain: + assert body["native"] == {"stringValue": "v"} + assert "cannot construct policy output" not in log_text(service) + finally: + service.stop() + + +def test_duplicate_resource_attributes_collapse_on_edit(tmp_path): + """Setting a duplicated resource attribute exports a single value.""" + fts = {"name": "fts", "code": '''function process(log) + log.context.attributes:get("resource"):get("attributes"):set("service.name", "a") + end'''} + service = start_otlp_json_service(tmp_path, [fts]) + try: + start, end = event_time(0xFFFFFFFF), event_time(0xFFFFFFFE) + now = event_time(1700000000) + attributes = raw_map([("service.name", "a"), ("service.name", "b")]) + entries = [[[start, {"schema": "otlp", "resource_id": 0, "scope_id": 0}], + {"resource": {"attributes": attributes}, "scope": {"name": "s"}}], + [[now, {"otlp": {}}], {"msg": "x"}], + [[end, {}], {}]] + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=30) as connection: + connection.sendall(pack(["stream", entries])) + resources = service.wait_for_condition(lambda: otlp_resources(service), timeout=20, + description="exported resource") + assert resources[0][0] == [("service.name", {"stringValue": "a"})] + assert "policy failed" not in log_text(service) + finally: + service.stop() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py b/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py new file mode 100644 index 00000000000..0be7dcf8975 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_security_batches.py @@ -0,0 +1,254 @@ +"""Privacy policies must execute for every item, despite malformed siblings.""" +import base64 +import copy +import json +import re +import time +from pathlib import Path + +import pytest +import requests +import yaml +from google.protobuf.json_format import ParseDict +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port +from test_fts_signals import attr, metric, span +from test_fts_policies import attributes, scalar + +POLICY = ''' +local fb = require("fluentbit") +if type(getmetatable("")) ~= "string" then error("string metatable exposed") end +assert = function() end +require = function() return {} end +fb._attach = function() return function() end end +fb._validate = function() end +fb._fts_wrap = function() return {} end +fb._fts_validate_telemetry = function() end +fb._fts_equal = function() return true end +math.huge = 0 +string.match = function() return true end +function process(record) + if record.body ~= nil then + fb.redact(record.body, {"password", "secret"}) + if record.body:get_string("action") == "bad" then record.metadata:ensure_map("otlp"):set("severity_number", fb.int64(999)) end + elseif record.signal == "metrics" then + for i=1,record.datapoints:len() do + fb.redact(record.datapoints:get(i):get("attributes"), {"secret", "password"}) + end + if record.name == "bad" then record.datapoints:get(1):set("time_unix_nano", fb.uint64(0)) end + else + fb.redact(record.attributes, {"secret", "password"}) + if record.name == "bad" then record.status:set("code", fb.int64(7)) end + end +end +''' + + +class BatchRunner: + def __init__(self, manager, port, signal): + self.manager, self.port, self.signal = manager, port, signal + self.sequence = 0 + self.log_offset = 0 + self.keys = {"logs": ("resourceLogs", "scopeLogs", "logRecords"), + "metrics": ("resourceMetrics", "scopeMetrics", "metrics"), + "traces": ("resourceSpans", "scopeSpans", "spans")}[signal] + + def send(self, items, transport="protobuf", scopes=None, resource_attributes=None): + self.log_offset = len(Path(self.manager.log_file).read_text()) + self.sequence += 1 + request = str(self.sequence) + resource_key, scope_key, item_key = self.keys + payload = {resource_key: [{"resource": {"attributes": [attr("test.request", request)] + (resource_attributes or [])}, + scope_key: scopes or [{"scope": {"name": "audit"}, item_key: items}]}]} + if transport == "protobuf": + def convert(value): + if isinstance(value, dict): + return {k: base64.b64encode(bytes.fromhex(v)).decode() if k in + ("traceId", "spanId", "parentSpanId") else convert(v) for k, v in value.items()} + if isinstance(value, list): + return [convert(v) for v in value] + return value + message = {"logs": ExportLogsServiceRequest, "metrics": ExportMetricsServiceRequest, + "traces": ExportTraceServiceRequest}[self.signal]() + ParseDict(convert(payload), message) + response = requests.post(f"http://127.0.0.1:{self.port}/v1/{self.signal}", + data=message.SerializeToString(), headers={"Content-Type": "application/x-protobuf"}, timeout=30) + else: + response = requests.post(f"http://127.0.0.1:{self.port}/v1/{self.signal}", json=payload, timeout=30) + assert response.status_code in (200, 201), response.text + expected = sum(len(s[item_key]) for s in scopes) if scopes else len(items) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + resources = [] + text = Path(self.manager.log_file).read_text() + # Diagnostics can interrupt a large stdout write. Remove complete log + # messages before reconstructing the JSON artifact, retaining partial writes. + text = re.sub(r"\[\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d+\] " + r"\[\s*(?:error|warn|info|debug|trace)\][^\n]*\n", "", text) + for line in text.splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + try: + batch = json.loads(line) + except json.JSONDecodeError: + continue + for resource in batch.get(resource_key, []): + if attributes(resource.get("resource", {}).get("attributes", [])).get("test.request") == request: + resources.append(resource) + rows = [item for r in resources for s in r[scope_key] for item in s[item_key]] + if len(rows) >= expected: + return resources, rows + time.sleep(0.1) + raise AssertionError(Path(self.manager.log_file).read_text()[-5000:]) + + + def recent_log(self): + return Path(self.manager.log_file).read_text()[self.log_offset:] + + +@pytest.fixture(scope="module", params=[(s, p) for s in ("logs", "metrics", "traces") + for p in ("input", "output")]) +def batch_runner(request, tmp_path_factory): + signal, placement = request.param + port = find_available_port() + source = {"name": "opentelemetry", "port": port, "buffer_max_size": "8M"} + output = {"name": "stdout", "match": "*", "format": "otlp_json"} + (source if placement == "input" else output)["processors"] = { + signal: [{"name": "fts", "code": POLICY, "error_mode": "keep", "memory_limit": "4M"}]} + config = tmp_path_factory.mktemp("fts-security-batches") / "config.yaml" + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, "pipeline": {"inputs": [source], "outputs": [output]}})) + manager = FluentBitManager(str(config)) + try: + manager.start() + yield BatchRunner(manager, port, signal) + finally: + manager.stop() + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_duplicate_parent_redaction(batch_runner, transport): + if batch_runner.signal == "metrics": + item = metric("gauge", "privacy") + values = item["gauge"]["dataPoints"][0]["attributes"] + elif batch_runner.signal == "traces": + item = span("privacy") + values = item["attributes"] + else: + item = {"timeUnixNano": "2147483648123456789", "body": {"kvlistValue": {"values": []}}} + values = item["body"]["kvlistValue"]["values"] + values.extend([{"key": "a", "value": {"kvlistValue": {"values": [attr("password", text)]}}} + for text in ("first-secret", "second-secret")]) + _, rows = batch_runner.send([item], transport) + rendered = json.dumps(rows) + assert "first-secret" not in rendered and "second-secret" not in rendered + assert rendered.count("[REDACTED]") >= 2 + assert "cannot construct FTS" not in batch_runner.recent_log() + + +def test_validation_is_private_and_item_local(batch_runner): + if batch_runner.signal == "metrics": + items = [metric("gauge", "bad"), metric("gauge", "good")] + elif batch_runner.signal == "traces": + items = [span("bad"), span("good")] + else: + items = [{"severityNumber": 2, "body": {"kvlistValue": {"values": [attr("action", action), + attr("secret", "sensitive")]}}} for action in ("bad", "good")] + _, rows = batch_runner.send(items) + assert len(rows) == 2 + if batch_runner.signal == "metrics": + points = [row["gauge"]["dataPoints"][0] for row in rows] + assert points[0]["timeUnixNano"] == "2147483648123456789" + assert attributes(points[0]["attributes"])["secret"] == "sensitive" + assert attributes(points[1]["attributes"])["secret"] == "[REDACTED]" + elif batch_runner.signal == "traces": + assert rows[0]["status"]["code"] == 1 + assert attributes(rows[0]["attributes"])["secret"] == "sensitive" + assert attributes(rows[1]["attributes"])["secret"] == "[REDACTED]" + else: + assert rows[0]["severityNumber"] == 2 + assert scalar(rows[0]["body"])["secret"] == "sensitive" + assert scalar(rows[1]["body"])["secret"] == "[REDACTED]" + log = batch_runner.recent_log() + assert "edits rolled back" in log and "cannot construct FTS" not in log + + +def test_unknown_inherited_kind_does_not_bypass_privacy(batch_runner): + if batch_runner.signal != "traces": + pytest.skip("trace enum preservation") + items = [span("unknown"), span("good")] + items[0]["kind"] = 9 + items[0]["parentSpanId"] = "0" * 16 + items[0]["endTimeUnixNano"] = "1" + _, rows = batch_runner.send(items) + # the span is kept; stored traces record an unknown kind as unspecified + assert len(rows) == 2 and rows[0].get("kind", 0) == 0 + assert all(attributes(r["attributes"])["secret"] == "[REDACTED]" for r in rows) + assert "edits rolled back" not in batch_runner.recent_log() + + +def test_large_trace_batch_is_redacted(batch_runner): + if batch_runner.signal != "traces": + pytest.skip("trace batch memory accounting") + items = [span(f"span-{i}") for i in range(4000)] + for item in items: + item["attributes"] += [attr(f"extra-{i}", "value") for i in range(3)] + resources, rows = batch_runner.send(items, resource_attributes=[attr("bulk", "b" * 32768)]) + assert len(rows) == 4000 and len(resources) == 1 + assert len(resources[0]["scopeSpans"]) == 1 + assert all(attributes(r["attributes"])["secret"] == "[REDACTED]" for r in rows) + assert "cannot construct FTS" not in batch_runner.recent_log() + + +@pytest.mark.parametrize("separate_scopes", [False, True]) +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_repeated_metric_names_are_preserved(batch_runner, separate_scopes, transport): + if batch_runner.signal != "metrics": + pytest.skip("metric family identity") + items = [metric("gauge", "same"), metric("gauge", "same")] + items[1]["gauge"]["dataPoints"][0]["asInt"] = "42" + items[0]["metadata"] = [attr("family.id", "first")] + items[1]["metadata"] = [attr("family.id", "second")] + scopes = [{"scope": {"name": name}, "metrics": [item]} for name, item in zip(("one", "two"), items)] + resources, rows = batch_runner.send(items, transport, scopes=scopes if separate_scopes else None) + assert len(rows) == 2 + assert [int(r["gauge"]["dataPoints"][0]["asInt"]) for r in rows] == [8, 42] + assert [attributes(r["metadata"])["family.id"] for r in rows] == ["first", "second"] + assert all(attributes(r["gauge"]["dataPoints"][0]["attributes"])["secret"] == "[REDACTED]" for r in rows) + if separate_scopes: + assert [s["scope"]["name"] for r in resources for s in r["scopeMetrics"]] == ["one", "two"] + assert "cannot construct FTS" not in batch_runner.recent_log() + + +def test_snapshot_limit_is_item_local(batch_runner): + if batch_runner.signal != "traces": + pytest.skip("trace snapshot isolation") + items = [span("oversized"), span("good")] + items[0]["attributes"] += [attr(f"field-{i}", "value") for i in range(9000)] + _, rows = batch_runner.send(items) + assert len(rows) == 2 + assert attributes(rows[0]["attributes"])["secret"] == "sensitive" + assert attributes(rows[1]["attributes"])["secret"] == "[REDACTED]" + text = batch_runner.recent_log() + assert "snapshot limits" in text and "cannot construct FTS" not in text + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_untouched_invalid_metric_does_not_skip_redaction(batch_runner, transport): + if batch_runner.signal != "metrics": + pytest.skip("metric field validation") + item = metric("gauge", "invalid-inherited-time") + point = item["gauge"]["dataPoints"][0] + point["startTimeUnixNano"] = "2147483648999999999" + point["timeUnixNano"] = "2147483648000000000" + _, rows = batch_runner.send([item], transport) + result = rows[0]["gauge"]["dataPoints"][0] + assert result["startTimeUnixNano"] == point["startTimeUnixNano"] + assert result["timeUnixNano"] == point["timeUnixNano"] + assert attributes(result["attributes"])["secret"] == "[REDACTED]" + assert "policy failed" not in batch_runner.recent_log() + assert "cannot construct FTS" not in batch_runner.recent_log() diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_signals.py b/tests/integration/scenarios/processor_fts/tests/test_fts_signals.py new file mode 100644 index 00000000000..0c07928a52d --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_signals.py @@ -0,0 +1,471 @@ +"""FTS metrics and traces policies based on OTTL transformation workflows.""" +import base64 +import copy +import json +import time +from pathlib import Path + +import pytest +import requests +import yaml +from google.protobuf.json_format import ParseDict +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port +from test_fts_policies import attributes + + +POLICY = r''' +local fb = require("fluentbit") +function process(record) + local case = record.name + if case == "control" or case == "noop" then return end + if case == "rename" then + record.name = "normalized"; record.resource.attributes:set("service.name", "normalized") + record.scope.name = "normalized.scope"; record.scope.version = "2" + record.resource.schema_url = "https://example.com/resource" + record.scope.schema_url = "https://example.com/scope" + elseif case == "resource" then + record.resource.attributes:set("deployment.environment.name", "production") + record.resource.attributes:remove("secret") + record.scope.attributes:set("policy.version", fb.int64(2)) + elseif case == "tag" then + assert(type(record.tag) == "string" and #record.tag > 0) + record.resource.attributes:set("source.tag", record.tag) + elseif case == "module_guard" then + assert(module == nil and os == nil and debug == nil and package == nil) + assert(not pcall(function() module("os") end)) + assert(not pcall(require, "debug")) + record.name = "restricted" + elseif case == "drop" then return fb.DROP + elseif case == "rollback" then + record.name = "partial"; record.resource.attributes:set("secret", "partial") + error("rollback-test") + elseif case == "bad_name" then record.name = fb.map() + elseif case == "bad_utf8" then record.name = string.char(255) + elseif case == "bad_nul" then record.name = "a\0b" + elseif case == "bad_resource" then record.resource.attributes:set("invalid", fb.uint64("18446744073709551615")) + elseif case == "bad_scope" then record.scope.attributes = fb.array() + elseif case == "bad_return" then return true + elseif case == "budget" then while true do end + elseif record.signal == "metrics" then + if case == "metadata" then + record.description = "normalized description"; record.unit = "s" + elseif case == "labels" then + for i=1,record.datapoints:len() do + local attrs = record.datapoints:get(i):get("attributes") + attrs:set("region", "west"); attrs:remove("secret") + end + elseif case == "values" then + fb.set_metric_value(record.datapoints:get(1), fb.int64("-9223372036854775808")) + elseif case == "scale" then fb.scale_metric(record, 2, "s") + elseif case == "scale_fraction" then fb.scale_metric(record, 0.5, "s") + elseif case == "to_sum" then + record.type = "sum"; record.aggregation_temporality = "delta"; record.is_monotonic = false + elseif case == "to_gauge" then record.type = "gauge" + elseif case == "copy" then + local derived = fb.copy_metric(record, "derived", "copy", "ms") + derived.resource.attributes:set("service.name", "derived") + elseif case == "extract_count" then fb.extract_metric(record, "count", "derived") + elseif case == "extract_sum" then fb.extract_metric(record, "sum", "derived") + elseif case == "extract_avg" then fb.extract_metric(record, "avg", "derived") + elseif case:match("^aggregate_") then + fb.aggregate_metric(record, {"region"}, case:sub(11)) + elseif case == "merge_buckets" then + local bounds = fb.array(); bounds:append(2.0) + fb.merge_histogram_buckets(record, bounds) + elseif case == "bad_value" then fb.set_metric_value(record.datapoints:get(1), fb.map()) + elseif case == "typed_labels" then + local attrs = record.datapoints:get(1):get("attributes") + attrs:set("typed", fb.int64("9223372036854775807")); attrs:set("enabled", true) + attrs:set("binary", fb.bytes("\0\255")) + local nested = fb.map(); nested:set("count", fb.int64(2)); attrs:set("nested", nested) + elseif case == "bad_label" then + record.datapoints:get(1):get("attributes"):set("typed", fb.uint64("18446744073709551615")) + elseif case == "bad_time" then record.datapoints:get(1):set("time_unix_nano", fb.uint64(0)) + end + else + if case == "redact" then + fb.redact(record.attributes, {"secret"}) + fb.redact(record.events, {"secret"}); fb.redact(record.links, {"secret"}) + elseif case == "status" then record.status:set("code", fb.int64(2)); record.status:set("message", "failed") + elseif case == "kind" then record.kind = 3 + elseif case == "correlation" then + record.trace_id = "1234567890abcdef1234567890abcdef" + record.span_id = "1234567890abcdef"; record.parent_span_id = "" + elseif case == "time" then record.end_time_unix_nano = fb.uint64("2147483648999999999") + elseif case == "events" then + local events = fb.array(); local event = fb.map() + event:set("name", "normalized.event") + event:set("time_unix_nano", fb.uint64("2147483648123456789")) + event:set("attributes", fb.map()); event:set("dropped_attributes_count", fb.uint64(0)) + events:append(event); record.events = events + elseif case == "bad_id" then record.trace_id = "00000000000000000000000000000000" + elseif case == "bad_time" then record.end_time_unix_nano = fb.uint64(0) + elseif case == "bad_status" then record.status:set("code", fb.int64("9223372036854775807")) + end + end +end +''' + + +def attr(key, value): + return {"key": key, "value": {"stringValue": value}} + + +def number_point(value=8): + return {"attributes": [attr("region", "east"), attr("secret", "sensitive")], + "startTimeUnixNano": "2147483648000000000", "timeUnixNano": "2147483648123456789", + "asInt": str(value), "flags": 1} + + +def metric(kind, name): + point = number_point() + if kind == "histogram": + point.pop("asInt") + point.update(count="3", sum=4.0, bucketCounts=["1", "1", "1"], explicitBounds=[1.0, 2.0], + min=0.5, max=2.5) + elif kind == "exponentialHistogram": + point.pop("asInt") + point.update(count="3", sum=2.0, scale=0, zeroCount="1", zeroThreshold=0.0, + positive={"offset": 0, "bucketCounts": ["1"]}, + negative={"offset": 0, "bucketCounts": ["1"]}, min=-1.0, max=2.0) + elif kind == "summary": + point.pop("asInt") + point.update(count="3", sum=4.0, quantileValues=[{"quantile": 0.5, "value": 1.5}, + {"quantile": 0.9, "value": 2.5}]) + point.pop("flags") + data = {"dataPoints": [point]} + if kind in ("sum", "histogram", "exponentialHistogram"): + data["aggregationTemporality"] = 2 + if kind == "sum": + data["isMonotonic"] = True + return {"name": name, "description": "original description", "unit": "ms", kind: data} + + +def span(name): + return {"name": name, "traceId": "0102030405060708090a0b0c0d0e0f10", + "spanId": "0102030405060708" if name != "control" else "0807060504030201", + "parentSpanId": "1112131415161718", "traceState": "vendor=value", "kind": 2, + "startTimeUnixNano": "2147483648000000000", "endTimeUnixNano": "2147483648123456789", + "attributes": [attr("secret", "sensitive"), {"key": "exact", "value": {"intValue": "9223372036854775807"}}], + "events": [{"timeUnixNano": "2147483648123456789", "name": "original.event", + "attributes": [attr("secret", "sensitive")], "droppedAttributesCount": 4}], + "links": [{"traceId": "0102030405060708090a0b0c0d0e0f10", "spanId": "1112131415161718", + "traceState": "vendor=link", "attributes": [attr("secret", "sensitive")], + "droppedAttributesCount": 5}], + "status": {"code": 1, "message": "ok"}, "droppedAttributesCount": 1, + "droppedEventsCount": 2, "droppedLinksCount": 3} + + +class SignalRunner: + def __init__(self, manager, port, signal): + self.manager, self.port, self.signal = manager, port, signal + self.sequence = 0 + + def run(self, case, kind="gauge", transport="protobuf", points=None, temporality=None, + extra_resource=None): + self.log_offset = len(Path(self.manager.log_file).read_text()) + self.sequence += 1 + request_id = f"{case}-{self.sequence}" + metrics = self.signal == "metrics" + resource_key, scope_key, item_key = (("resourceMetrics", "scopeMetrics", "metrics") if metrics else + ("resourceSpans", "scopeSpans", "spans")) + items = [metric(kind, case), metric(kind, "control")] if metrics else [span(case), span("control")] + if points is not None: + items[0][kind]["dataPoints"] = points + if temporality is not None: + items[0][kind]["aggregationTemporality"] = temporality + payload = {resource_key: [{"resource": {"attributes": [attr("test.request", request_id), + attr("secret", "sensitive"), attr("service.name", "original")] + (extra_resource or []), + "droppedAttributesCount": 7}, + "schemaUrl": "https://example.com/original-resource", + scope_key: [{"scope": {"name": "original.scope", "version": "1", + "attributes": [{"key": "exact", "value": {"intValue": "9223372036854775807"}}], + "droppedAttributesCount": 6}, "schemaUrl": "https://example.com/original-scope", + item_key: items}]}]} + if transport == "json": + response = requests.post(f"http://127.0.0.1:{self.port}/v1/{self.signal}", json=payload, timeout=10) + else: + def convert(value): + if isinstance(value, dict): + return {k: base64.b64encode(bytes.fromhex(v)).decode() if k in + ("traceId", "spanId", "parentSpanId") else convert(v) for k, v in value.items()} + if isinstance(value, list): + return [convert(v) for v in value] + return value + message = ExportMetricsServiceRequest() if metrics else ExportTraceServiceRequest() + ParseDict(convert(payload), message) + response = requests.post(f"http://127.0.0.1:{self.port}/v1/{self.signal}", + data=message.SerializeToString(), headers={"Content-Type": "application/x-protobuf"}, timeout=10) + assert response.status_code in (200, 201), response.text + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + rows = [] + for line in Path(self.manager.log_file).read_text().splitlines(keepends=True): + if not line.startswith("{") or not line.endswith("\n"): + continue + batch = json.loads(line) + for resource in batch.get(resource_key, []): + if attributes(resource.get("resource", {}).get("attributes", [])).get("test.request") != request_id: + continue + for scope in resource.get(scope_key, []): + rows += [(resource, scope, item) for item in scope.get(item_key, [])] + if rows: + if not case.startswith("bad_") and case not in ("rollback", "budget"): + diagnostics = Path(self.manager.log_file).read_text()[self.log_offset:] + assert "cannot construct" not in diagnostics + assert "policy failed" not in diagnostics + return items[0], rows + time.sleep(0.1) + raise AssertionError(f"no {self.signal} output for {case}: {Path(self.manager.log_file).read_text()[-5000:]}") + + +@pytest.fixture(scope="module", params=[("metrics", "input"), ("metrics", "output"), + ("traces", "input"), ("traces", "output")]) +def signal_runner(request, tmp_path_factory): + signal, placement = request.param + config = tmp_path_factory.mktemp(f"fts-{signal}-{placement}") / "signals.yaml" + port = find_available_port() + source = {"name": "opentelemetry", "port": port} + output = {"name": "stdout", "match": "*", "format": "otlp_json", "workers": 2} + (source if placement == "input" else output)["processors"] = { + signal: [{"name": "fts", "code": POLICY, "error_mode": "keep"}]} + config.write_text(yaml.safe_dump({"service": {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [source], "outputs": [output]}})) + manager = FluentBitManager(str(config)) + try: + manager.start() + yield SignalRunner(manager, port, signal) + finally: + manager.stop() + + +COMMON_CASES = ["noop", "rename", "resource", "tag", "module_guard", "drop", "rollback", "bad_name", "bad_utf8", "bad_nul", + "bad_resource", "bad_scope", "bad_return", "budget"] +METRIC_CASES = ["metadata", "labels", "typed_labels", "values", "scale", "scale_fraction", "to_sum", "to_gauge", "copy", + "extract_count", "extract_sum", "extract_avg", "bad_value", "bad_label", "bad_time"] +TRACE_CASES = ["redact", "status", "kind", "correlation", "time", "events", "bad_id", "bad_time", "bad_status"] + + +@pytest.mark.parametrize("case", list(dict.fromkeys(COMMON_CASES + METRIC_CASES + TRACE_CASES))) +def test_signal_policy_matrix(signal_runner, case): + signal = signal_runner.signal + if case not in COMMON_CASES and case not in (METRIC_CASES if signal == "metrics" else TRACE_CASES): + pytest.skip("case belongs to the other signal") + kind = "sum" if case == "to_gauge" else "histogram" if case.startswith("extract_") else "gauge" + original, rows = signal_runner.run(case, kind) + control = [(r, s, item) for r, s, item in rows if item["name"] == "control"] + assert len(control) == 1 + resource, scope, item = control[0] + assert attributes(resource["resource"]["attributes"])["service.name"] == "original" + assert scope["scope"]["name"] == "original.scope" + assert scope["scope"]["droppedAttributesCount"] == 6 + targets = [(r, s, item) for r, s, item in rows if item["name"] != "control"] + if case == "drop": + assert not targets + return + assert targets + resource, scope, item = targets[0] + if case.startswith("bad_") or case in ("rollback", "budget", "noop"): + assert item["name"] == case + assert attributes(resource["resource"]["attributes"])["service.name"] == "original" + assert scope["scope"]["name"] == "original.scope" + elif case == "rename": + assert item["name"] == "normalized" + assert attributes(resource["resource"]["attributes"])["service.name"] == "normalized" + assert scope["scope"]["name"] == "normalized.scope" and scope["scope"]["version"] == "2" + assert resource["schemaUrl"] == "https://example.com/resource" + assert scope["schemaUrl"] == "https://example.com/scope" + elif case == "resource": + attrs = attributes(resource["resource"]["attributes"]) + assert "secret" not in attrs and attrs["deployment.environment.name"] == "production" + assert attributes(scope["scope"]["attributes"])["policy.version"] == 2 + elif case == "tag": + assert attributes(resource["resource"]["attributes"])["source.tag"] + elif case == "module_guard": + assert item["name"] == "restricted" + elif signal == "metrics": + data = item.get("gauge") or item.get("sum") or item.get("histogram") + point = data["dataPoints"][0] + if case == "metadata": + assert item["description"] == "normalized description" and item["unit"] == "s" + elif case == "labels": + assert attributes(point["attributes"])["region"] == "west" + assert "secret" not in attributes(point["attributes"]) + elif case == "values": + assert int(point["asInt"]) == -(2**63) + elif case == "typed_labels": + attrs = attributes(point["attributes"]) + assert attrs["typed"] == 2**63-1 and attrs["enabled"] is True + assert attrs["nested"] == {"count": 2} + assert any(a["key"] == "binary" and a["value"]["bytesValue"] == "AP8=" + for a in point["attributes"]) + elif case in ("scale", "scale_fraction"): + assert float(point.get("asInt", point.get("asDouble"))) == (16 if case == "scale" else 4) + assert item["unit"] == "s" + elif case == "to_sum": + assert "sum" in item and item["sum"]["aggregationTemporality"] == 1 + assert not item["sum"]["isMonotonic"] + elif case == "to_gauge": + assert "gauge" in item and "sum" not in item + elif case == "copy" or case.startswith("extract_"): + derived = [(r, s, m) for r, s, m in targets if m["name"] == "derived"] + assert len(derived) == 1 + r, _, m = derived[0] + if case == "copy": + assert attributes(r["resource"]["attributes"])["service.name"] == "derived" + assert m["description"] == "copy" and m["unit"] == "ms" + else: + p = (m.get("gauge") or m.get("sum"))["dataPoints"][0] + assert float(p.get("asInt", p.get("asDouble"))) == {"extract_count": 3, "extract_sum": 4, + "extract_avg": 4/3}[case] + else: + assert attributes(item["attributes"])["exact"] == 2**63-1 + if case == "redact": + assert attributes(item["attributes"])["secret"] == "[REDACTED]" + assert attributes(item["events"][0]["attributes"])["secret"] == "[REDACTED]" + assert attributes(item["links"][0]["attributes"])["secret"] == "[REDACTED]" + elif case == "status": + assert item["status"] == {"code": 2, "message": "failed"} + elif case == "kind": + assert item["kind"] == 3 + elif case == "correlation": + assert item["traceId"] == "1234567890abcdef1234567890abcdef" + assert item["spanId"] == "1234567890abcdef" + assert item.get("parentSpanId", "") == "" + elif case == "time": + assert int(item["endTimeUnixNano"]) == 2147483648999999999 + elif case == "events": + assert len(item["events"]) == 1 and item["events"][0]["name"] == "normalized.event" + + +@pytest.mark.parametrize("kind", ["gauge", "sum", "histogram", "exponentialHistogram", "summary"]) +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_metric_types_round_trip(signal_runner, kind, transport): + if signal_runner.signal != "metrics": + pytest.skip("metrics case") + _, rows = signal_runner.run("noop", kind, transport) + target = next(item for _, _, item in rows if item["name"] == "noop") + point = target[kind]["dataPoints"][0] + assert int(point["timeUnixNano"]) == 2147483648123456789 + assert int(point["startTimeUnixNano"]) == 2147483648000000000 + if kind in ("gauge", "sum"): + assert int(point["asInt"]) == 8 + else: + assert int(point["count"]) == 3 + assert point["sum"] == (2.0 if kind == "exponentialHistogram" else 4.0) + + +@pytest.mark.parametrize("kind", ["gauge", "sum", "histogram", "exponentialHistogram", "summary"]) +# OTLP JSON input merges repeated series by labels before FTS runs, like 5.1. +@pytest.mark.parametrize("transport", ["protobuf"]) +def test_typed_series_keep_independent_metadata(signal_runner, kind, transport): + if signal_runner.signal != "metrics": + pytest.skip("metric datapoint semantics") + points = [copy.deepcopy(metric(kind, "noop")[kind]["dataPoints"][0]) for _ in range(2)] + points[0]["attributes"].append({"key": "typed", "value": {"intValue": "1"}}) + points[1]["attributes"].append(attr("typed", "1")) + if kind != "summary": + points[0]["flags"], points[1]["flags"] = 7, 3 + _, rows = signal_runner.run("noop", kind, transport, points) + target = next(item for _, _, item in rows if item["name"] == "noop") + exported = target[kind]["dataPoints"] + assert len(exported) == 2 + assert [attributes(point["attributes"])["typed"] for point in exported] == [1, "1"] + if kind != "summary": + assert [point["flags"] for point in exported] == [7, 3] + assert not any(entry["key"] in {"resource_index", "scope_index", "scope_flat_index"} + for entry in target.get("metadata", [])) + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_bytes_context_attributes_round_trip(signal_runner, transport): + raw = {"key": "binary", "value": {"bytesValue": base64.b64encode(b"\x00\x01\xff").decode()}} + _, rows = signal_runner.run("noop", "gauge", transport, extra_resource=[raw]) + resource = rows[0][0]["resource"]["attributes"] + exported = next(entry["value"] for entry in resource if entry["key"] == "binary") + assert base64.b64decode(exported["bytesValue"]) == b"\x00\x01\xff" + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_repeated_unlabeled_points_keep_last_value(signal_runner, transport): + """Like 5.1, a repeated label-less series keeps its last value.""" + if signal_runner.signal != "metrics": + pytest.skip("metric datapoint semantics") + points = [number_point(1), number_point(2)] + for index, point in enumerate(points): + point["attributes"] = [] + point["timeUnixNano"] = str(2147483648123456789 + index) + _, rows = signal_runner.run("noop", "gauge", transport, points) + target = next(item for _, _, item in rows if item["name"] == "noop") + assert [int(point["asInt"]) for point in target["gauge"]["dataPoints"]] == [2] + + +@pytest.mark.parametrize("operation,expected", [("sum", 20), ("min", 8), ("max", 12), + ("mean", 10), ("count", 2)]) +def test_metric_cardinality_aggregation(signal_runner, operation, expected): + if signal_runner.signal != "metrics": + pytest.skip("metric aggregation") + points = [number_point(8), number_point(12)] + for index, point in enumerate(points): + point["attributes"].append(attr("method", ["GET", "POST"][index])) + _, rows = signal_runner.run(f"aggregate_{operation}", "gauge", points=points) + target = next(item for _, _, item in rows if item["name"] != "control") + exported = target["gauge"]["dataPoints"] + assert len(exported) == 1 + assert float(exported[0].get("asInt", exported[0].get("asDouble"))) == expected + assert attributes(exported[0]["attributes"]) == {"region": "east"} + + +def test_histogram_bucket_merge(signal_runner): + if signal_runner.signal != "metrics": + pytest.skip("metric bucket semantics") + _, rows = signal_runner.run("merge_buckets", "histogram") + target = next(item for _, _, item in rows if item["name"] != "control") + point = target["histogram"]["dataPoints"][0] + assert point["explicitBounds"] == [2.0] + assert [int(value) for value in point["bucketCounts"]] == [2, 1] + assert int(point["count"]) == 3 and point["sum"] == 4.0 + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_scaling_includes_exemplars(signal_runner, transport): + if signal_runner.signal != "metrics": + pytest.skip("metric scaling") + point = number_point() + point["exemplars"] = [{"asDouble": 3.0, "timeUnixNano": point["timeUnixNano"]}] + _, rows = signal_runner.run("scale", transport=transport, points=[point]) + target = next(item for _, _, item in rows if item["name"] == "scale") + result = target["gauge"]["dataPoints"][0] + assert int(result["asInt"]) == 16 + assert result["exemplars"][0]["asDouble"] == 6.0 + + +# OTLP JSON input merges repeated series by labels before FTS runs, like 5.1. +@pytest.mark.parametrize("transport", ["protobuf"]) +def test_aggregation_preserves_distinct_start_times(signal_runner, transport): + if signal_runner.signal != "metrics": + pytest.skip("metric windows") + points = [number_point(8), number_point(42)] + points[0]["startTimeUnixNano"] = "2147483647000000000" + _, rows = signal_runner.run("aggregate_sum", transport=transport, points=points) + target = next(item for _, _, item in rows if item["name"] == "aggregate_sum") + result = target["gauge"]["dataPoints"] + assert len(result) == 2 + assert [p["startTimeUnixNano"] for p in result] == [p["startTimeUnixNano"] for p in points] + assert [int(p["asInt"]) for p in result] == [8, 42] + assert all(attributes(p["attributes"]) == {"region": "east"} for p in result) + + +@pytest.mark.parametrize("transport", ["json", "protobuf"]) +def test_extraction_preserves_unspecified_temporality(signal_runner, transport): + if signal_runner.signal != "metrics": + pytest.skip("metric extraction") + _, rows = signal_runner.run("extract_sum", "histogram", transport=transport, temporality=0) + target = next(item for _, _, item in rows if item["name"] == "derived") + assert target["sum"].get("aggregationTemporality", 0) == 0 + assert target["sum"]["dataPoints"][0]["asDouble"] == 4.0 diff --git a/tests/integration/scenarios/processor_fts/tests/test_fts_signals_pipeline.py b/tests/integration/scenarios/processor_fts/tests/test_fts_signals_pipeline.py new file mode 100644 index 00000000000..064a21806f4 --- /dev/null +++ b/tests/integration/scenarios/processor_fts/tests/test_fts_signals_pipeline.py @@ -0,0 +1,119 @@ +"""FTS metric/span policies through protobuf export, fan-out, retries and replay.""" +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +import requests +import yaml +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from utils.test_service import FluentBitTestService +from test_fts_signals import metric, span, attr +from test_fts_policies import attributes + + +@pytest.mark.parametrize("signal", ["metrics", "traces"]) +@pytest.mark.parametrize("filesystem,replay", [(False, False), (True, False), (True, True)]) +def test_signal_route_isolation_retry_and_backlog(tmp_path, signal, filesystem, replay): + message_type = ExportMetricsServiceRequest if signal == "metrics" else ExportTraceServiceRequest + attempts, received = [], [] + blocked = {"value": replay} + + class Collector(BaseHTTPRequestHandler): + def do_POST(self): + data = self.rfile.read(int(self.headers["Content-Length"])) + attempts.append(data) + message = message_type.FromString(data) + status = 503 if blocked["value"] or len(attempts) == 1 else 200 + if status == 200: + received.append(message) + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Collector) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + policy = '''local fb = require("fluentbit") +function process(record) + if record.name == "control" then return end + record.name = "normalized" + record.resource.attributes:set("service.name", "normalized") + record.scope.name = "normalized.scope" + if record.signal == "metrics" then + record.datapoints:get(1):get("attributes"):set("exact", fb.int64("9223372036854775807")) + else + record.attributes:remove("secret") + record.attributes:set("exact", fb.int64("9223372036854775807")) + end +end''' + source = {"name": "opentelemetry", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}"} + service_config = {"flush": 0.2, "grace": 1, "http_server": True, + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}", + "scheduler.base": 1, "scheduler.cap": 1} + if filesystem: + source["storage.type"] = "filesystem" + service_config["storage.path"] = str(tmp_path / "storage") + config = tmp_path / "pipeline.yaml" + config.write_text(yaml.safe_dump({"service": service_config, "pipeline": { + "inputs": [source], "outputs": [ + {"name": "stdout", "match": "*", "format": "otlp_json"}, + {"name": "opentelemetry", "match": "*", "host": "127.0.0.1", "port": server.server_port, + "retry_limit": False, "processors": {signal: [{"name": "fts", "code": policy}]}} + ]}})) + service = FluentBitTestService(str(config)) + try: + service.start() + metrics = signal == "metrics" + resource_key, scope_key, item_key = (("resourceMetrics", "scopeMetrics", "metrics") if metrics else + ("resourceSpans", "scopeSpans", "spans")) + items = [metric("gauge", "target"), metric("gauge", "control")] if metrics else [span("target"), span("control")] + payload = {resource_key: [{"resource": {"attributes": [attr("service.name", "original")], + "droppedAttributesCount": 7}, scope_key: [{"scope": {"name": "original.scope", + "droppedAttributesCount": 6}, item_key: items}]}]} + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/{signal}", json=payload, timeout=15) + assert response.status_code in (200, 201), response.text + if replay: + service.wait_for_condition(lambda: attempts, timeout=30) + service.stop() + original_log = Path(service.flb.log_file).read_text() + assert list((tmp_path / "storage").rglob("*.flb")) + blocked["value"] = False + service = FluentBitTestService(str(config)) + service.start() + service.wait_for_condition(lambda: received, timeout=30) + service.stop() + if not replay: + original_log = Path(service.flb.log_file).read_text() + assert len(attempts) >= 2 and attempts[0] == attempts[1] + rows = [(resource, scope, item) for message in received + for resource in getattr(message, "resource_metrics" if metrics else "resource_spans") + for scope in getattr(resource, "scope_metrics" if metrics else "scope_spans") + for item in getattr(scope, "metrics" if metrics else "spans")] + assert [item.name for _, _, item in rows] == ["normalized", "control"] + for resource, scope, item in rows: + values = {entry.key: entry.value.string_value for entry in resource.resource.attributes} + assert values["service.name"] == ("normalized" if item.name == "normalized" else "original") + assert scope.scope.name == ("normalized.scope" if item.name == "normalized" else "original.scope") + assert resource.resource.dropped_attributes_count == 7 + assert scope.scope.dropped_attributes_count == 6 + if item.name == "normalized": + attrs = item.gauge.data_points[0].attributes if metrics else item.attributes + assert next(entry.value.int_value for entry in attrs if entry.key == "exact") == 2**63-1 + batches = [json.loads(line) for line in original_log.splitlines() if line.startswith("{")] + raw = [(resource, scope, item) for batch in batches for resource in batch[resource_key] + for scope in resource[scope_key] for item in scope[item_key]] + assert [item["name"] for _, _, item in raw] == ["target", "control"] + assert all(attributes(resource["resource"]["attributes"])["service.name"] == "original" + and scope["scope"]["name"] == "original.scope" for resource, scope, _ in raw) + finally: + service.stop() + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 93d079f7b2b..6ef8190c122 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -216,6 +216,10 @@ if(FLB_IN_OPENTELEMETRY) list(APPEND UNIT_TESTS_FILES opentelemetry_protobuf.c) endif() +if(FLB_LUAJIT AND FLB_YYJSON AND FLB_PROCESSOR_FTS) + list(APPEND UNIT_TESTS_FILES fts.c) +endif() + set(UNIT_TESTS_DATA data/fips/disabled.cnf data/tls/certificate.pem diff --git a/tests/internal/fts.c b/tests/internal/fts.c new file mode 100644 index 00000000000..383b9f32153 --- /dev/null +++ b/tests/internal/fts.c @@ -0,0 +1,1515 @@ +/* Fluent Bit: native log-policy regression tests, Apache-2.0. */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "flb_tests_internal.h" +#include +#include +#include "../../plugins/processor_fts/fts.h" + +static msgpack_object *field(msgpack_object *map, const char *key) +{ + size_t i; + + if (!map || map->type != MSGPACK_OBJECT_MAP) { + return NULL; + } + for (i = 0; i < map->via.map.size; i++) { + if (map->via.map.ptr[i].key.type == MSGPACK_OBJECT_STR && + map->via.map.ptr[i].key.via.str.size == strlen(key) && + memcmp(map->via.map.ptr[i].key.via.str.ptr, key, strlen(key)) == 0) { + return &map->via.map.ptr[i].val; + } + } + return NULL; +} + +static void run_policy(const char *code, int grouped, const char *error_mode, + int expected_records, int expected_groups, int conditional, + const char *limit_key, const char *limit) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + struct flb_time ts; + msgpack_object *value; + void *output = NULL; + size_t output_size = 0; + int i; + int type; + int records = 0; + int groups = 0; + int ret; + int depth = 0; + int dup_count; + const char *kind; + const char zero_id[16] = {0}; + msgpack_object empty_map = {.type = MSGPACK_OBJECT_MAP, .via.map = {0, NULL}}; + msgpack_object array_values[2] = { + {.type = MSGPACK_OBJECT_STR, .via.str = {1, "\xff"}}, + {.type = MSGPACK_OBJECT_STR, .via.str = {2, "ok"}} + }; + msgpack_object legacy_array = {.type = MSGPACK_OBJECT_ARRAY, .via.array = {2, array_values}}; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "lua_policy_test", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", code) == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "error_mode", error_mode) == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "instruction_limit", limit_key ? "1000000" : "10000") == 0); + if (limit_key) { + TEST_CHECK(flb_processor_unit_set_property_str(unit, limit_key, limit) == 0); + } + if (conditional) { + unit->condition = flb_condition_create(FLB_COND_OP_AND); + TEST_ASSERT(unit->condition != NULL); + TEST_CHECK(flb_condition_add_rule(unit->condition, "$kind", FLB_RULE_OP_EQ, + "selected", 0, RECORD_CONTEXT_BODY) == FLB_TRUE); + } + TEST_ASSERT(flb_processor_init(processor) == 0); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (i = 0; i < 6; i++) { + if (grouped && i % 3 == 0) { + TEST_CHECK(flb_log_event_encoder_group_init(encoder) == 0); + if (grouped == 3 || grouped >= 5) { + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("batch"), FLB_LOG_EVENT_CSTRING_VALUE("original"), + FLB_LOG_EVENT_CSTRING_VALUE("schema"), FLB_LOG_EVENT_CSTRING_VALUE("otlp"), + FLB_LOG_EVENT_CSTRING_VALUE("resource_id"), FLB_LOG_EVENT_INT64_VALUE(INT64_MAX), + FLB_LOG_EVENT_CSTRING_VALUE("scope_id"), FLB_LOG_EVENT_INT64_VALUE(INT64_MIN)); + } + else { + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("batch"), FLB_LOG_EVENT_CSTRING_VALUE("original"), + FLB_LOG_EVENT_CSTRING_VALUE("schema"), FLB_LOG_EVENT_BOOLEAN_VALUE(FLB_FALSE)); + } + TEST_CHECK(ret == 0); + if (grouped == 5) { + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("duplicate"), FLB_LOG_EVENT_CSTRING_VALUE("one"), + FLB_LOG_EVENT_CSTRING_VALUE("duplicate"), FLB_LOG_EVENT_CSTRING_VALUE("two")); + TEST_CHECK(ret == 0); + } + if (grouped == 6) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("empty"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&empty_map)); + TEST_CHECK(ret == 0); + } + if (grouped == 11) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("duplicate"), FLB_LOG_EVENT_CSTRING_VALUE("one"), + FLB_LOG_EVENT_CSTRING_VALUE("duplicate"), FLB_LOG_EVENT_CSTRING_VALUE("two")); + TEST_CHECK(ret == 0); + } + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("producer"), FLB_LOG_EVENT_CSTRING_VALUE("original")); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_group_header_end(encoder) == 0); + } + if (grouped == 10 && i == 1) { + TEST_CHECK(flb_log_event_encoder_group_init(encoder) == 0); + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("batch"), FLB_LOG_EVENT_CSTRING_VALUE("original"), + FLB_LOG_EVENT_CSTRING_VALUE("schema"), FLB_LOG_EVENT_BOOLEAN_VALUE(FLB_FALSE)); + TEST_CHECK(ret == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("producer"), FLB_LOG_EVENT_CSTRING_VALUE("original")); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_group_header_end(encoder) == 0); + } + kind = i % 2 ? "other" : "selected"; + TEST_CHECK(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(&ts, 1700000000 + i, i); + TEST_CHECK(flb_log_event_encoder_set_timestamp(encoder, &ts) == 0); + if (grouped == 3 || grouped >= 5) { + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("severity_number"), FLB_LOG_EVENT_INT64_VALUE(30), + FLB_LOG_EVENT_CSTRING_VALUE("trace_id"), FLB_LOG_EVENT_BINARY_VALUE(zero_id, sizeof(zero_id))); + TEST_CHECK(ret == 0); + } + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("index"), FLB_LOG_EVENT_INT64_VALUE(i), + FLB_LOG_EVENT_CSTRING_VALUE("kind"), + FLB_LOG_EVENT_CSTRING_VALUE(kind), + FLB_LOG_EVENT_CSTRING_VALUE("secret"), FLB_LOG_EVENT_CSTRING_VALUE("one"), + FLB_LOG_EVENT_CSTRING_VALUE("secret"), FLB_LOG_EVENT_CSTRING_VALUE("two")); + TEST_CHECK(ret == 0); + if (grouped == 3 || grouped >= 5) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("legacy_uint"), FLB_LOG_EVENT_UINT64_VALUE(UINT64_MAX)); + TEST_CHECK(ret == 0); + } + if (grouped == 9) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("legacy_array"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&legacy_array)); + TEST_CHECK(ret == 0); + } + if (grouped == 7) { + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("duplicate_utf8"), FLB_LOG_EVENT_CSTRING_VALUE("valid"), + FLB_LOG_EVENT_CSTRING_VALUE("duplicate_utf8"), FLB_LOG_EVENT_STRING_VALUE("\xff", 1)); + TEST_CHECK(ret == 0); + } + TEST_CHECK(flb_log_event_encoder_commit_record(encoder) == 0); + if (grouped && i % 3 == 2 && !(grouped == 2 && i == 5)) { + TEST_CHECK(flb_log_event_encoder_group_end(encoder) == 0); + if (grouped == 10 && i == 2) { + TEST_CHECK(flb_log_event_encoder_group_end(encoder) == 0); + } + } + } + ret = flb_processor_run(processor, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, &output, &output_size); + TEST_CHECK(ret == 0); + if (output_size > 0) { + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_CHECK(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type == FLB_LOG_EVENT_GROUP_START) { + TEST_CHECK(depth == 0); + depth++; + groups++; + value = field(event.metadata, "batch"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_STR && + value->via.str.size == 8 && memcmp(value->via.str.ptr, "original", 8) == 0); + if (grouped == 6) { + TEST_CHECK(field(event.body, "empty") == NULL); + } + if (grouped == 11) { + /* the edit left a single copy of the duplicated key */ + dup_count = 0; + for (i = 0; i < (int) event.body->via.map.size; i++) { + value = &event.body->via.map.ptr[i].key; + if (value->via.str.size == 9 && + memcmp(value->via.str.ptr, "duplicate", 9) == 0) { + dup_count++; + value = &event.body->via.map.ptr[i].val; + TEST_CHECK(value->type == MSGPACK_OBJECT_STR && + value->via.str.size == 3 && + memcmp(value->via.str.ptr, "one", 3) == 0); + } + } + TEST_CHECK(dup_count == 1); + } + if (grouped == 8) { + TEST_CHECK(field(event.body, "resource") == NULL); + TEST_CHECK(field(event.body, "scope") == NULL); + } + value = field(event.body, "producer"); + TEST_ASSERT(value != NULL); + if (expected_groups == 6) { + TEST_CHECK(value->type == MSGPACK_OBJECT_POSITIVE_INTEGER && value->via.u64 == records); + } + else { + TEST_CHECK(value->type == MSGPACK_OBJECT_STR && value->via.str.size == 8 && + memcmp(value->via.str.ptr, "original", 8) == 0); + } + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + TEST_CHECK(depth == 1); + depth--; + } + else { + TEST_CHECK(depth == !!grouped); + value = field(event.body, "index"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_POSITIVE_INTEGER); + if (expected_records == 6) { + TEST_CHECK(value->via.u64 == records); + } + if (conditional) { + TEST_CHECK(value->via.u64 == 2 * records + 1); + } + if (strcmp(error_mode, "keep") == 0) { + TEST_CHECK(field(event.body, "grow") == NULL); + TEST_CHECK(field(event.body, "verified") == NULL); + TEST_CHECK(event.body->via.map.size == (grouped == 3 ? 5 : 4)); + } + if (grouped == 3 || grouped >= 5) { + value = field(event.body, "legacy_uint"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->type == MSGPACK_OBJECT_POSITIVE_INTEGER && value->via.u64 == UINT64_MAX); + value = field(event.metadata, "severity_number"); + TEST_ASSERT(value != NULL); + TEST_CHECK(value->via.u64 == 30); + } + if (grouped == 8) { + value = field(event.metadata, "otlp"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_MAP); + TEST_CHECK(field(value, "attributes") == NULL); + TEST_CHECK(field(value, "event_name") != NULL); + } + if (grouped == 9) { + value = field(event.body, "legacy_array"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_ARRAY); + TEST_ASSERT(value->via.array.size == 3); + TEST_CHECK(value->via.array.ptr[0].type == MSGPACK_OBJECT_STR); + TEST_CHECK(value->via.array.ptr[0].via.str.size == 3); + TEST_CHECK(memcmp(value->via.array.ptr[0].via.str.ptr, "new", 3) == 0); + TEST_CHECK(value->via.array.ptr[1].via.str.size == 1); + TEST_CHECK((unsigned char) value->via.array.ptr[1].via.str.ptr[0] == 255); + } + records++; + if (field(event.body, "verified")) { + TEST_CHECK(field(event.body, "secret") == NULL); + TEST_CHECK(event.body->via.map.size == (grouped == 3 ? 4 : 3)); + } + } + } + /* a group left open by the input stays open for the next append */ + TEST_CHECK(depth == (grouped == 2 ? 1 : 0)); + flb_log_event_decoder_destroy(&decoder); + } + TEST_CHECK(records == expected_records); + TEST_CHECK(groups == expected_groups); + if (output && output != encoder->output_buffer) { + flb_free(output); + } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void generic_context_isolation(void) +{ + run_policy("function process(log) " + "assert(log.resource == nil and log.scope == nil and log.attributes == nil); " + "assert(log.context.attributes:get_string('producer') == 'original'); " + "assert(log.context.metadata:get_string('batch') == 'original'); " + "log.context.attributes:set('producer', log.body:get('index')); " + "log.body:remove('secret'); log.body:set('verified', true) end", + 1, "drop", 6, 6, 0, NULL, NULL); +} + +static void generic_read_does_not_split(void) +{ + run_policy("function process(log) assert(log.context.metadata:get_string('batch') == 'original') end", + 1, "drop", 6, 2, 0, NULL, NULL); +} + +static void plain_logs_do_not_gain_context(void) +{ + run_policy("function process(log) assert(log.context == nil and log.resource == nil); " + "log.body:remove('secret'); log.body:set('verified', true) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void conditional_drop_preserves_groups(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) return fb.DROP end", + 1, "drop", 3, 2, 1, NULL, NULL); +} + +static void drop_all_removes_empty_groups(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) return fb.DROP end", + 1, "drop", 0, 0, 0, NULL, NULL); +} + +static void duplicate_redaction_canonicalizes(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "fb.redact(log.body, {'secret'}); assert(log.body:len() == 3); " + "log.body:remove('secret'); log.body:set('verified', true) end", + 1, "drop", 6, 2, 0, NULL, NULL); +} + +static void errors_restore_body_and_context(void) +{ + run_policy("function process(log) log.body:remove('secret'); " + "log.context.attributes:set('producer','partial'); error('failure') end", + 1, "keep", 6, 2, 0, NULL, NULL); +} + +static void instruction_exhaustion_drops(void) +{ + run_policy("function process(log) while true do end end", + 1, "drop", 0, 0, 0, NULL, NULL); +} + +static void caught_budget_errors_terminate(void) +{ + run_policy("function process(log) while true do pcall(function() while true do end end) end end", + 0, "drop", 0, 0, 0, NULL, NULL); +} + +static void vm_memory_exhaustion_drops(void) +{ + run_policy("function process(log) local s=string.rep('x', 2000000); log.body:set('large',s) end", + 1, "drop", 0, 0, 0, "memory_limit", "1M"); +} + +static void native_memory_exhaustion_drops(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) local a=fb.array(); " + "for i=1,5000 do a:append(fb.int64(i)) end end", + 1, "drop", 0, 0, 0, "memory_limit", "1M"); +} + +static void strict_parsing_and_time(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "assert(fb.parse_json('18446744073709551615'):value() == '18446744073709551615'); " + "assert(fb.parse_json('-9223372036854775808'):value() == '-9223372036854775808'); " + "assert(not pcall(fb.parse_json, '{\"a\":1,\"a\":2}')); " + "assert(not pcall(fb.parse_json, '{} garbage')); " + "assert(not pcall(fb.parse_json, '18446744073709551616')); " + "assert(fb.parse_time('1970-01-01T01:00:00+01:00').sec == 0); " + "assert(fb.parse_time('2024-02-29T00:00:00.000000001Z').nsec == 1); " + "assert(not pcall(fb.parse_time,'2100-02-29T00:00:00Z')) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void output_growth_fails_closed(void) +{ + run_policy("function process(log) log.body:set('grow', string.rep('x',300000)) end", + 1, "drop", 0, 0, 0, "output_limit", "1M"); +} + +static void output_growth_keep_restores_whole_chunk(void) +{ + run_policy("function process(log) log.body:set('grow', string.rep('x',300000)) end", + 1, "keep", 6, 2, 0, "output_limit", "1M"); +} + +static void initialization_failures(void) +{ + const char *codes[] = {"while true do end", "function !syntax", "error('init failure')", + "function other() end", "require('ffi')", "io.open('/tmp/x')", NULL}; + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + int i; + + for (i = 0; codes[i]; i++) { + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "init_failure", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", codes[i]) == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "instruction_limit", "10000") == 0); + TEST_CHECK(flb_processor_init(processor) != 0); + flb_processor_destroy(processor); + flb_config_exit(config); + } +} + +static void unclosed_group_preserves_records(void) +{ + run_policy("function process(log) end", 2, "drop", 6, 2, 0, NULL, NULL); +} + +static void exact_opaque_context_ids(void) +{ + run_policy("function process(log) " + "assert(log.context.metadata:get('resource_id'):value() == '9223372036854775807'); " + "assert(log.context.metadata:get('scope_id'):value() == '-9223372036854775808'); " + "assert(log.resource and log.scope); " + "log.body:remove('secret'); log.body:set('verified',true) end", + 3, "drop", 6, 2, 0, NULL, NULL); +} + +static void context_replacement_during_collection(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local replacement=fb.map(); " + "for _,key in ipairs(log.context.metadata:keys()) do " + "replacement:set(key,log.context.metadata:get(key)) end; " + "log.body:remove('secret'); log.context.metadata=replacement; collectgarbage() end", + 1, "keep", 6, 2, 0, NULL, NULL); + run_policy("local fb=require('fluentbit'); function process(log) " + "log.body:remove('secret'); log.context.attributes=fb.map(); collectgarbage() end", + 1, "keep", 6, 2, 0, NULL, NULL); +} + +static void deep_mutable_views(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) local node=log.body; " + "for i=1,60 do node=node:ensure_map('child') end; " + "node:set('value',fb.int64(1)); assert(node:get_number('value')==1); " + "log.body:remove('child'); log.body:remove('secret'); " + "log.body:set('verified',true) end", + 1, "drop", 6, 2, 0, NULL, NULL); +} + +static void garbage_collector_is_private(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "assert(log.body.__gc == nil and fb.null.__gc == nil) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void trusted_policy_jit_compiles(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_processor_instance *instance; + struct fts_runtime *runtime; + lua_State *l; + const char *probe = "return jit.status(), require('jit.util').traceinfo(1)"; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_jit_test", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "jit", "true") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "instruction_limit", "0") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", + "local sum=0; for j=1,30 do for i=1,100000 do sum=sum+i end end; " + "function process(log) end") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + instance = unit->ctx; + runtime = instance->context; + l = runtime->vm->state; + TEST_CHECK(lua_gethookmask(l) == 0); + TEST_ASSERT(luaL_loadbuffer(l, probe, strlen(probe), "jit_test") == 0); + lua_rawgeti(l, LUA_REGISTRYINDEX, runtime->helpers_env_ref); + lua_setfenv(l, -2); + TEST_ASSERT(lua_pcall(l, 0, 2, 0) == 0); + TEST_CHECK(lua_toboolean(l, -2)); + TEST_CHECK(lua_istable(l, -1)); + lua_pop(l, 2); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void native_histogram_buckets_are_preserved(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct cmt *input; + struct cmt *output; + struct cmt_histogram *histogram; + struct cmt_histogram_buckets *buckets; + void *result; + size_t result_size; + int edit; + + for (edit = 0; edit < 2; edit++) { + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_histogram_test", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_METRICS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", edit ? + "function process(metric) metric.name = 'renamed' end" : + "function process(metric) end") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + input = cmt_create(); + TEST_ASSERT(input != NULL); + buckets = cmt_histogram_buckets_create(2, 4.0, 7.0); + TEST_ASSERT(buckets != NULL); + histogram = cmt_histogram_create(input, "", "", "native", "native", buckets, 0, NULL); + TEST_ASSERT(histogram != NULL); + TEST_CHECK(cmt_histogram_observe(histogram, 100, 2.0, 0, NULL) == 0); + TEST_CHECK(cmt_histogram_observe(histogram, 100, 5.0, 0, NULL) == 0); + TEST_CHECK(cmt_histogram_observe(histogram, 100, 8.0, 0, NULL) == 0); + result = NULL; + result_size = 0; + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_METRICS, "native", 6, + input, 0, &result, &result_size) == 0); + output = result; + TEST_ASSERT(output != NULL); + TEST_ASSERT(cfl_list_size(&output->histograms) == 1); + histogram = cfl_list_entry_first(&output->histograms, struct cmt_histogram, _head); + TEST_CHECK(strcmp(histogram->opts.name, edit ? "renamed" : "native") == 0); + TEST_CHECK(histogram->map->metric.hist_buckets[0] == 1); + TEST_CHECK(histogram->map->metric.hist_buckets[1] == 2); + TEST_CHECK(histogram->map->metric.hist_buckets[2] == 3); + TEST_CHECK(cmt_metric_hist_get_count_value(&histogram->map->metric) == 3); + TEST_CHECK(cmt_metric_hist_get_sum_value(&histogram->map->metric) == 15.0); + if (output != input) { + cmt_destroy(output); + } + cmt_destroy(input); + flb_processor_destroy(processor); + flb_config_exit(config); + } +} + +static void native_metric_scalar_preservation(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct cmt *input; + struct cmt *output; + struct cmt_gauge *gauge; + struct cmt_gauge *sibling; + struct cfl_list *head; + int found; + void *result; + size_t result_size; + int mode; + + for (mode = 0; mode < 4; mode++) { + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_metric_test", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_METRICS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", + "function process(metric) metric.name = 'renamed' end") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "error_mode", mode == 2 ? "drop" : "keep") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + input = cmt_create(); + TEST_ASSERT(input != NULL); + gauge = cmt_gauge_create(input, "", "", "native", "native", 0, NULL); + TEST_ASSERT(gauge != NULL); + gauge->map->metric_static_set = CMT_TRUE; + if (mode == 0) { + cmt_metric_set_int64(&gauge->map->metric, 100, INT64_MIN); + } + else if (mode == 3) { + /* uint64 values that fit int64 still run the policy. */ + cmt_metric_set_uint64(&gauge->map->metric, 100, 5); + } + else { + cmt_metric_set_uint64(&gauge->map->metric, 100, UINT64_MAX); + sibling = cmt_gauge_create(input, "", "", "healthy", "healthy", 0, NULL); + TEST_ASSERT(sibling != NULL); + TEST_ASSERT(cmt_gauge_set(sibling, 100, 42, 0, NULL) == 0); + } + result = NULL; + result_size = 0; + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_METRICS, "native", 6, + input, 0, &result, &result_size) == 0); + output = result; + TEST_ASSERT(output != NULL); + if (mode == 0) { + gauge = cfl_list_entry_first(&output->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + TEST_CHECK(strcmp(gauge->opts.name, "renamed") == 0); + TEST_CHECK(cmt_metric_get_value_type(&gauge->map->metric) == CMT_METRIC_VALUE_INT64); + TEST_CHECK(cmt_metric_get_int64_value(&gauge->map->metric) == INT64_MIN); + } + else if (mode == 3) { + TEST_CHECK(cfl_list_size(&output->gauges) == 1); + gauge = cfl_list_entry_first(&output->gauges, struct cmt_gauge, _head); + TEST_CHECK(strcmp(gauge->opts.name, "renamed") == 0); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 5); + } + else if (mode == 1) { + TEST_CHECK(cfl_list_size(&output->gauges) == 2); + found = 0; + cfl_list_foreach(head, &output->gauges) { + sibling = cfl_list_entry(head, struct cmt_gauge, _head); + if (strcmp(sibling->opts.name, "native") == 0) { + found++; + TEST_CHECK(cmt_metric_get_value_type(&sibling->map->metric) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK(cmt_metric_get_uint64_value(&sibling->map->metric) == UINT64_MAX); + } + } + TEST_CHECK(found == 1); + } + else { + TEST_CHECK(cfl_list_size(&output->gauges) == 1); + } + if (mode == 1) { + /* Kept families are copied: the producer still owns its context. */ + TEST_CHECK(output != input); + TEST_CHECK(cfl_list_size(&input->gauges) == 2); + } + if (mode == 1 || mode == 2) { + found = 0; + cfl_list_foreach(head, &output->gauges) { + sibling = cfl_list_entry(head, struct cmt_gauge, _head); + if (strcmp(sibling->opts.name, "renamed") == 0) { + TEST_CHECK(cmt_metric_get_value(&sibling->map->metric) == 42); + found++; + } + } + TEST_CHECK(found == 1); + } + if (output != input) { + cmt_destroy(output); + } + cmt_destroy(input); + flb_processor_destroy(processor); + flb_config_exit(config); + } +} + +static void kept_native_family_does_not_merge_into_outputs(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct cmt *input; + struct cmt *output; + struct cmt_gauge *native; + struct cmt_gauge *healthy; + struct cmt_gauge *gauge; + struct cfl_list *head; + void *result; + size_t result_size; + int seen_native = 0; + int seen_healthy = 0; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_metric_collision", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_METRICS, "fts"); + TEST_ASSERT(unit != NULL); + /* the policy output reuses the name of a family kept outside the policy */ + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", + "function process(metric) metric.name = 'native' end") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "error_mode", "keep") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + + input = cmt_create(); + TEST_ASSERT(input != NULL); + native = cmt_gauge_create(input, "", "", "native", "shared", 0, NULL); + TEST_ASSERT(native != NULL); + native->map->metric_static_set = CMT_TRUE; + cmt_metric_set_uint64(&native->map->metric, 100, UINT64_MAX); + healthy = cmt_gauge_create(input, "", "", "healthy", "shared", 0, NULL); + TEST_ASSERT(healthy != NULL); + TEST_ASSERT(cmt_gauge_set(healthy, 100, 42, 0, NULL) == 0); + + result = NULL; + result_size = 0; + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_METRICS, "native", 6, + input, 0, &result, &result_size) == 0); + output = result; + TEST_ASSERT(output != NULL); + + /* the batch falls back to its original families instead of merging them */ + TEST_CHECK(cfl_list_size(&output->gauges) == 2); + cfl_list_foreach(head, &output->gauges) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + if (strcmp(gauge->opts.name, "native") == 0) { + seen_native++; + TEST_CHECK(cmt_metric_get_value_type(&gauge->map->metric) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK(cmt_metric_get_uint64_value(&gauge->map->metric) == UINT64_MAX); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 0); + } + else if (strcmp(gauge->opts.name, "healthy") == 0) { + seen_healthy++; + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 42); + } + } + TEST_CHECK(seen_native == 1 && seen_healthy == 1); + + if (output != input) { + cmt_destroy(output); + } + cmt_destroy(input); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void untouched_otel_fields_are_preserved(void) +{ + run_policy("function process(log) end", 3, "drop", 6, 2, 0, NULL, NULL); + run_policy("function process(log) log.body:set('edited', true) end", 3, "drop", 6, 2, 0, NULL, NULL); +} + +static void schema_promotion_is_record_local(void) +{ + const char *policy = "function process(log) if log.body:get('index'):number() == 0 then " + "log.context.metadata:set('schema', 'otlp') end end"; + + run_policy(policy, 1, "keep", 6, 2, 0, NULL, NULL); + run_policy(policy, 1, "drop", 5, 2, 0, NULL, NULL); +} + +static void reserved_timestamps_are_record_errors(void) +{ + run_policy("function process(log) if log.body:get('index'):number() == 0 then " + "log.timestamp.sec = 4294967294; log.timestamp.nsec = 1 end end", + 0, "keep", 6, 0, 0, NULL, NULL); + run_policy("function process(log) if log.body:get('index'):number() == 0 then " + "log.timestamp.sec = 4294967295; log.timestamp.nsec = 0 end end", + 0, "drop", 5, 0, 0, NULL, NULL); +} + +static void small_record_batch_uses_wire_budget(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + struct flb_time timestamp; + void *output; + size_t output_size; + int index; + int count; + int ret; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "fts_wire_budget", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", "function process(log) end") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + flb_time_set(×tamp, 1, 0); + for (index = 0; index < 75000; index++) { + TEST_CHECK(flb_log_event_encoder_begin_record(encoder) == 0); + TEST_CHECK(flb_log_event_encoder_set_timestamp(encoder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("value"), FLB_LOG_EVENT_INT64_VALUE((index % 10))); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_commit_record(encoder) == 0); + } + output = NULL; + output_size = 0; + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_LOGS, "wire", 4, + encoder->output_buffer, encoder->output_length, &output, &output_size) == 0); + TEST_ASSERT(output != NULL); + TEST_CHECK(output_size < 3 * 1024 * 1024); + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + count = 0; + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { count++; } + TEST_CHECK(count == 75000); + flb_log_event_decoder_destroy(&decoder); + if (output != encoder->output_buffer) { flb_free(output); } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void event_metatable_cannot_bypass_validation(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local context=log.context; local metadata=log.metadata; local n=0; " + "rawset(log,'context',nil); setmetatable(log,{__index=function(_,key) " + "if key=='context' then n=n+1; if n==6 then " + "local otlp=metadata:ensure_map('otlp'); " + "otlp:set('severity_number',fb.int64(999)); otlp:set('trace_id',fb.bytes('abc')); " + "end; return context end end}) end", + 3, "drop", 0, 0, 0, NULL, NULL); +} + +static void replaced_and_redacted_values_do_not_accumulate_charges(void) +{ + run_policy("local fb=require('fluentbit'); local shared=fb.map(); " + "function process(log) for i=1,2000 do " + "shared:set('secret',true); fb.redact(shared,{'secret'}); " + "shared:remove('secret') end; " + "log.body:remove('secret'); log.body:set('verified',true) end", + 0, "drop", 6, 0, 0, "memory_limit", "1M"); +} + +static int exhaust_or_recover_memory(lua_State *l) +{ + static char payload[300000]; + char entry[1024]; + size_t index; + + if (lua_touserdata(l, 1) == NULL) { + lua_gc(l, LUA_GCSTOP, 0); + lua_newtable(l); + for (index = 0; index < 10000; index++) { + memset(entry, 'x', sizeof(entry)); + snprintf(entry, 32, "%zu", index); + lua_pushlstring(l, entry, sizeof(entry)); + lua_rawseti(l, -2, index + 1); + } + return luaL_error(l, "expected memory exhaustion"); + } + memset(payload, 'x', sizeof(payload)); + lua_pushlstring(l, payload, sizeof(payload)); + return 1; +} + +static void memory_exhaustion_does_not_poison_later_callbacks(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_processor_instance *instance; + struct fts_runtime *runtime; + lua_State *l; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "memory_recovery", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "code", "function process(log) end") == 0); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "memory_limit", "1M") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + instance = unit->ctx; + runtime = instance->context; + l = runtime->vm->state; + luaL_unref(l, LUA_REGISTRYINDEX, runtime->callback_ref); + lua_pushcfunction(l, exhaust_or_recover_memory); + runtime->callback_ref = luaL_ref(l, LUA_REGISTRYINDEX); + TEST_CHECK(fts_runtime_call(runtime, NULL, 1) == LUA_ERRMEM); + lua_settop(l, 0); + TEST_CHECK(fts_runtime_call(runtime, runtime, 1) == 0); + TEST_CHECK(lua_objlen(l, -1) == 300000); + lua_settop(l, 0); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void policies_cannot_stop_automatic_gc(void) +{ + run_policy("function process(log) " + "assert(not pcall(collectgarbage,'stop')); " + "log.body:remove('secret'); log.body:set('verified',true) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void duplicate_context_metadata_does_not_split_groups(void) +{ + run_policy("function process(log) end", 5, "drop", 6, 2, 0, NULL, NULL); +} + +static void removing_empty_context_field_is_preserved(void) +{ + run_policy("function process(log) log.context.attributes:remove('empty') end", + 6, "drop", 6, 2, 0, NULL, NULL); +} + +static void inherited_duplicate_invalid_utf8_is_preserved(void) +{ + run_policy("function process(log) log.body:set('edited',true) end", + 7, "drop", 6, 2, 0, NULL, NULL); +} + +static void json_source_limit_allows_large_scalar(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local text='\"'..string.rep('x',1048574)..'\"'; " + "assert(#fb.parse_json(text):value() == 1048574) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void documented_depth_is_enforced(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local text=string.rep('[',65)..string.rep(']',65); " + "assert(not pcall(fb.parse_json,text)) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void minimum_memory_initializes_all_signals(void) +{ + int signals[] = {FLB_PROCESSOR_LOGS, FLB_PROCESSOR_METRICS, FLB_PROCESSOR_TRACES}; + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + size_t index; + + for (index = 0; index < sizeof(signals) / sizeof(signals[0]); index++) { + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "minimum_memory", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, signals[index], "fts"); + TEST_ASSERT(unit != NULL); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "code", "function process(item) end") == 0); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "memory_limit", "262144") == 0); + TEST_CHECK(flb_processor_init(processor) == 0); + flb_processor_destroy(processor); + flb_config_exit(config); + } +} + +static void print_is_not_a_host_io_escape(void) +{ + run_policy("function process(log) assert(print==nil) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void json_duplicate_detection_is_byte_exact(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "assert(fb.parse_json([[{\"a\\u0000x\":1,\"a\\u0000y\":2,\"\":true}]]):len()==3); " + "assert(not pcall(fb.parse_json,[[{\"a\\u0000x\":1,\"a\\u0000x\":2}]])) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void log_edits_do_not_export_empty_read_placeholders(void) +{ + run_policy("function process(log) log.metadata:ensure_map('otlp'):set('event_name','changed') end", + 8, "drop", 6, 2, 0, NULL, NULL); +} + +static void mixed_integer_addition_preserves_exact_values(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local v=fb._fts_add(fb.int64(9007199254740991),fb.uint64(2)); " + "assert(v:type()=='int64' and v:value()=='9007199254740993'); " + "v=fb._fts_add(fb.int64('-9223372036854775808'),fb.uint64('18446744073709551615')); " + "assert(v:type()=='int64' and v:value()=='9223372036854775807'); " + "v=fb._fts_add(fb.int64('9223372036854775807'),fb.uint64(1)); " + "assert(v:type()=='uint64' and v:value()=='9223372036854775808'); " + "assert(not pcall(fb._fts_add,fb.int64(1),fb.uint64('18446744073709551615'))) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void invalid_mutations_do_not_accumulate_charges(void) +{ + run_policy("local fb=require('fluentbit'); local shared=fb.map(); " + "function process(log) for i=1,5000 do " + "assert(not pcall(function() shared:set('bad',{}) end)) end; " + "log.body:remove('secret'); log.body:set('verified',true) end", + 0, "drop", 6, 0, 0, "memory_limit", "1M"); +} + +static void native_nan_equality_preserves_unchanged_values(void) +{ + struct cfl_variant *left; + struct cfl_variant *right; + + left = cfl_variant_create_from_double(NAN); + TEST_ASSERT(left != NULL); + right = fts_copy_value(left, 0); + TEST_ASSERT(right != NULL); + TEST_CHECK(fts_values_equal(left, right, 0)); + cfl_variant_destroy(left); + cfl_variant_destroy(right); + left = cfl_variant_create_from_double(-0.0); + right = cfl_variant_create_from_double(0.0); + TEST_ASSERT(left != NULL && right != NULL); + TEST_CHECK(!fts_values_equal(left, right, 0)); + cfl_variant_destroy(left); + cfl_variant_destroy(right); +} + +static void array_insert_preserves_inherited_invalid_value(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local original=log.body:get('legacy_array'); local result=fb.array(); " + "result:append('new'); for i=1,original:len() do result:append(original:get(i)) end; " + "log.body:set('legacy_array',result) end", + 9, "drop", 6, 2, 0, NULL, NULL); +} + +static void nested_and_stray_group_markers_preserve_records(void) +{ + run_policy("function process(log) end", 10, "drop", 6, 3, 0, NULL, NULL); +} + +static void dropped_spans_remove_empty_resource_groups(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct ctrace *input; + struct ctrace_resource_span *resource; + struct ctrace_scope_span *scope; + void *output = NULL; + size_t output_size = 0; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "empty_traces", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_TRACES, "fts"); + TEST_ASSERT(unit != NULL); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "code", + "local fb=require('fluentbit'); function process(span) return fb.DROP end") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + input = ctr_create(NULL); + TEST_ASSERT(input != NULL); + resource = ctr_resource_span_create(input); + TEST_ASSERT(resource != NULL); + scope = ctr_scope_span_create(resource); + TEST_ASSERT(scope != NULL); + TEST_ASSERT(ctr_span_create(input, scope, "dropped", NULL) != NULL); + TEST_ASSERT(flb_processor_run(processor, 0, FLB_PROCESSOR_TRACES, "test", 4, + input, 0, &output, &output_size) == 0); + TEST_CHECK(output == input); + TEST_CHECK(cfl_list_is_empty(&input->span_list)); + TEST_CHECK(cfl_list_is_empty(&input->resource_spans)); + ctr_destroy(input); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void copied_values_preserve_binary_data_and_source_lifetime(void) +{ + struct cfl_kvlist *map; + struct cfl_variant *source; + struct cfl_variant *copy; + struct cfl_kvpair *pair; + char payload[] = {'a', '\0', '\xff'}; + + map = cfl_kvlist_create(); + TEST_ASSERT(map != NULL); + TEST_ASSERT(cfl_kvlist_insert_s(map, "string", 6, + cfl_variant_create_from_string_s(payload, sizeof(payload), CFL_TRUE)) == 0); + TEST_ASSERT(cfl_kvlist_insert_s(map, "bytes", 5, + cfl_variant_create_from_bytes(payload, sizeof(payload), CFL_TRUE)) == 0); + TEST_ASSERT(cfl_kvlist_insert_s(map, "empty", 5, + cfl_variant_create_from_string_s("", 0, CFL_TRUE)) == 0); + source = cfl_variant_create_from_kvlist(map); + TEST_ASSERT(source != NULL); + copy = fts_copy_value(source, 0); + TEST_ASSERT(copy != NULL); + memset(payload, 'x', sizeof(payload)); + cfl_variant_destroy(source); + pair = fts_find_pair(copy, "string", 6); + TEST_ASSERT(pair != NULL); + TEST_CHECK(pair->val->type == CFL_VARIANT_STRING); + TEST_CHECK(cfl_variant_size_get(pair->val) == 3); + TEST_CHECK(memcmp(pair->val->data.as_string, "a\0\xff", 3) == 0); + pair = fts_find_pair(copy, "bytes", 5); + TEST_ASSERT(pair != NULL); + TEST_CHECK(pair->val->type == CFL_VARIANT_BYTES); + TEST_CHECK(cfl_variant_size_get(pair->val) == 3); + TEST_CHECK(memcmp(pair->val->data.as_bytes, "a\0\xff", 3) == 0); + pair = fts_find_pair(copy, "empty", 5); + TEST_ASSERT(pair != NULL); + TEST_CHECK(cfl_variant_size_get(pair->val) == 0); + cfl_variant_destroy(copy); +} + +static void plain_log_fast_path_preserves_root_validation(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "log.metadata=fb.map() end", 0, "drop", 0, 0, 0, NULL, NULL); + run_policy("function process(log) log.context={} end", + 0, "drop", 0, 0, 0, NULL, NULL); + run_policy("function process(log) log.context.metadata:set('schema','otlp') end", + 1, "drop", 0, 0, 0, NULL, NULL); + run_policy("function process(log) log.metadata=nil end", + 0, "keep", 6, 0, 0, NULL, NULL); +} + +static void scalar_getters_preserve_types_and_live_paths(void) +{ + run_policy("local fb=require('fluentbit'); function process(log) " + "local m=fb.map(); m:set('s','a'..string.char(0)..'b'); m:set('b',false); " + "m:set('i',fb.int64('-9007199254740991')); " + "m:set('u',fb.uint64('9007199254740991')); m:set('n',fb.null); " + "assert(m:get_string('s')=='a'..string.char(0)..'b'); " + "assert(m:get_bool('b')==false and m:get_number('i')==-9007199254740991); " + "assert(m:get_number('u')==9007199254740991); " + "assert(m:get_string('missing')==nil and m:get_bool('missing')==nil " + "and m:get_number('missing')==nil); " + "for _,k in ipairs({'b','i','n'}) do assert(not pcall(m.get_string,m,k)) end; " + "assert(not pcall(m.get_bool,m,'s') and not pcall(m.get_number,m,'n')); " + "m:set('u',fb.uint64('9007199254740992')); " + "assert(not pcall(m.get_number,m,'u')); " + "m:set('i',fb.int64('-9007199254740992')); " + "assert(not pcall(m.get_number,m,'i')); " + "m:set('bytes',fb.bytes('abc')); assert(not pcall(m.get_string,m,'bytes')); " + "local a=fb.parse_json('[1,\"s\",true,null]'); " + "assert(a:get_number(1)==1 and a:get_string(2)=='s' and a:get_bool(3)); " + "for _,k in ipairs({0,5,1.5,math.huge,'1'}) do " + "assert(not pcall(a.get_number,a,k)) end; " + "assert(not pcall(a.get_string,a,4) and not pcall(m.get_string,m,1)); " + "local child=m:ensure_map('child'); child:set('s','before'); " + "m:set('child',fb.parse_json('{\"s\":\"after\"}')); " + "assert(child:get_string('s')=='after'); m:remove('child'); " + "assert(not pcall(child.get_string,child,'s')); " + "log.body:remove('secret'); log.body:set('verified',true) end", + 0, "drop", 6, 0, 0, NULL, NULL); +} + +static void native_result_arena_does_not_own_lua_snapshots(void) +{ + struct cfl_arena *arena; + struct cfl_variant *source; + struct cfl_variant *pooled; + struct cfl_variant *snapshot; + struct cfl_kvlist *map; + struct cfl_array *array; + char binary[] = {'a', '\0', (char) 0xff}; + + arena = cfl_arena_create(4096); + TEST_ASSERT(arena != NULL); + map = cfl_kvlist_create(); + array = cfl_array_create(2); + TEST_ASSERT(map != NULL && array != NULL); + TEST_ASSERT(cfl_kvlist_insert_bytes(map, "binary", binary, sizeof(binary), CFL_FALSE) == 0); + TEST_ASSERT(cfl_kvlist_insert_string(map, "duplicate", "first") == 0); + TEST_ASSERT(cfl_kvlist_insert_string(map, "duplicate", "second") == 0); + TEST_ASSERT(cfl_array_append_uint64(array, UINT64_MAX) == 0); + TEST_ASSERT(cfl_array_append_int64(array, INT64_MIN) == 0); + TEST_ASSERT(cfl_kvlist_insert_array(map, "array", array) == 0); + source = cfl_variant_create_from_kvlist(map); + TEST_ASSERT(source != NULL); + pooled = fts_copy_value_in(source, 0, arena); + TEST_ASSERT(pooled != NULL && pooled->arena == arena); + TEST_CHECK(fts_values_equal(source, pooled, 0)); + snapshot = fts_copy_value(pooled, 0); + TEST_ASSERT(snapshot != NULL && snapshot->arena == NULL); + cfl_variant_destroy(source); + TEST_CHECK(fts_values_equal(snapshot, pooled, 0)); + cfl_variant_destroy(pooled); + cfl_arena_destroy(arena); + TEST_CHECK(cfl_variant_size_get(fts_find_pair(snapshot, "binary", 6)->val) == sizeof(binary)); + TEST_CHECK(memcmp(fts_find_pair(snapshot, "binary", 6)->val->data.as_bytes, + binary, sizeof(binary)) == 0); + TEST_CHECK(snapshot->data.as_kvlist->arena == NULL); + cfl_variant_destroy(snapshot); +} + +/* A policy snapshot fits the minimum VM budget; a second commit tree does not. */ +static void edited_record_needs_one_policy_tree(void) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + struct flb_time timestamp; + msgpack_object *value; + char *payload; + void *output = NULL; + size_t output_size = 0; + int ret; + const char binary[] = {'a', '\0', (char) 0xff}; + msgpack_object empty_map = {.type = MSGPACK_OBJECT_MAP, .via.map = {0, NULL}}; + msgpack_object empty_array = {.type = MSGPACK_OBJECT_ARRAY, .via.array = {0, NULL}}; + + payload = flb_malloc(120000); + TEST_ASSERT(payload != NULL); + memset(payload, 'x', 120000); + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "single_snapshot", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "code", + "function process(log) log.body:set('edited',true) end") == 0); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "memory_limit", "262144") == 0); + TEST_ASSERT(flb_processor_unit_set_property_str(unit, "error_mode", "keep") == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + TEST_ASSERT(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(×tamp, 1700000000, 0); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(encoder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("payload"), FLB_LOG_EVENT_STRING_VALUE(payload, 120000), + FLB_LOG_EVENT_CSTRING_VALUE("binary"), FLB_LOG_EVENT_BINARY_VALUE(binary, sizeof(binary)), + FLB_LOG_EVENT_CSTRING_VALUE("unsigned"), FLB_LOG_EVENT_UINT64_VALUE(UINT64_MAX), + FLB_LOG_EVENT_CSTRING_VALUE("signed"), FLB_LOG_EVENT_INT64_VALUE(INT64_MIN), + FLB_LOG_EVENT_CSTRING_VALUE("zero"), FLB_LOG_EVENT_DOUBLE_VALUE(-0.0), + FLB_LOG_EVENT_CSTRING_VALUE("null"), FLB_LOG_EVENT_NULL_VALUE(), + FLB_LOG_EVENT_CSTRING_VALUE("map"), FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&empty_map), + FLB_LOG_EVENT_CSTRING_VALUE("array"), FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&empty_array)); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(encoder) == 0); + TEST_ASSERT(flb_processor_run(processor, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, &output, &output_size) == 0); + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + TEST_ASSERT(flb_log_event_decoder_next(&decoder, &event) == 0); + value = field(event.body, "edited"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_BOOLEAN && value->via.boolean); + value = field(event.body, "payload"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_STR && value->via.str.size == 120000); + value = field(event.body, "binary"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_BIN); + TEST_CHECK(value->via.bin.size == sizeof(binary)); + TEST_CHECK(memcmp(value->via.bin.ptr, binary, sizeof(binary)) == 0); + value = field(event.body, "unsigned"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_POSITIVE_INTEGER && + value->via.u64 == UINT64_MAX); + value = field(event.body, "signed"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_NEGATIVE_INTEGER && + value->via.i64 == INT64_MIN); + value = field(event.body, "zero"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_FLOAT64 && signbit(value->via.f64)); + value = field(event.body, "null"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_NIL); + value = field(event.body, "map"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_MAP && value->via.map.size == 0); + value = field(event.body, "array"); + TEST_CHECK(value != NULL && value->type == MSGPACK_OBJECT_ARRAY && value->via.array.size == 0); + flb_log_event_decoder_destroy(&decoder); + if (output != encoder->output_buffer) { + flb_free(output); + } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(processor); + flb_config_exit(config); + flb_free(payload); +} + +static void empty_metadata_is_materialized_independently(void) +{ + run_policy("local previous; function process(log) " + "if previous then previous:ensure_map('nested'):set('value','late') end; " + "assert(log.metadata:len()==0); " + "local set=log.metadata.set; assert(not pcall(set,{},'x',true)); " + "log.metadata:ensure_map('nested'):set('value','current'); " + "previous=log.metadata:copy() end", 0, "drop", 6, 0, 0, NULL, NULL); +} + +/* Group layouts split across appends: 'S' starts a group, 'E' ends one */ +static void append_split_group(struct flb_log_event_encoder *encoder, const char *name) +{ + int ret; + + TEST_CHECK(flb_log_event_encoder_group_init(encoder) == 0); + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("batch"), FLB_LOG_EVENT_CSTRING_VALUE(name), + FLB_LOG_EVENT_CSTRING_VALUE("schema"), FLB_LOG_EVENT_BOOLEAN_VALUE(FLB_FALSE)); + TEST_CHECK(ret == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("producer"), FLB_LOG_EVENT_CSTRING_VALUE(name)); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_group_header_end(encoder) == 0); +} + +static void append_split_record(struct flb_log_event_encoder *encoder, int index) +{ + int ret; + struct flb_time ts; + + TEST_CHECK(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(&ts, 1700000000 + index, 0); + TEST_CHECK(flb_log_event_encoder_set_timestamp(encoder, &ts) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("index"), FLB_LOG_EVENT_INT64_VALUE(index)); + TEST_CHECK(ret == 0); + TEST_CHECK(flb_log_event_encoder_commit_record(encoder) == 0); +} + +/* layout tokens: r record, S group start, E group end */ +static void run_split_layout(const char *code, const char *error_mode, + const char *layout, const char *expected) +{ + struct flb_config *config; + struct flb_processor *processor; + struct flb_processor_unit *unit; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + msgpack_object *value; + void *output = NULL; + size_t output_size = 0; + char result[256]; + char token[32]; + const char *cursor; + int type; + int length; + + config = flb_config_init(); + TEST_ASSERT(config != NULL); + processor = flb_processor_create(config, "split_groups", NULL, 0); + TEST_ASSERT(processor != NULL); + unit = flb_processor_unit_create(processor, FLB_PROCESSOR_LOGS, "fts"); + TEST_ASSERT(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "code", code) == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "error_mode", error_mode) == 0); + TEST_ASSERT(flb_processor_init(processor) == 0); + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (cursor = layout; *cursor; ) { + if (sscanf(cursor, "%31s%n", token, &length) != 1) { + break; + } + cursor += length; + if (token[0] == 'r') { + append_split_record(encoder, atoi(token + 1)); + } + else if (token[0] == 'S') { + append_split_group(encoder, token + 1); + } + else if (token[0] == 'E') { + TEST_CHECK(flb_log_event_encoder_group_end(encoder) == 0); + } + } + + TEST_CHECK(flb_processor_run(processor, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, + &output, &output_size) == 0); + + result[0] = '\0'; + if (output_size > 0) { + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type == FLB_LOG_EVENT_GROUP_START) { + value = field(event.body, "producer"); + TEST_ASSERT(value != NULL && value->type == MSGPACK_OBJECT_STR); + snprintf(token, sizeof(token), "S%.*s ", (int) value->via.str.size, + value->via.str.ptr); + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + snprintf(token, sizeof(token), "E "); + } + else { + value = field(event.body, "index"); + TEST_ASSERT(value != NULL); + snprintf(token, sizeof(token), "r%d ", (int) value->via.i64); + } + strncat(result, token, sizeof(result) - strlen(result) - 1); + } + flb_log_event_decoder_destroy(&decoder); + } + if (strlen(result) > 0) { + result[strlen(result) - 1] = '\0'; + } + TEST_CHECK(strcmp(result, expected) == 0); + TEST_MSG("layout '%s' produced '%s', expected '%s'", layout, result, expected); + + if (output && output != encoder->output_buffer) { + flb_free(output); + } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(processor); + flb_config_exit(config); +} + +static void groups_split_across_appends_are_preserved(void) +{ + const char *noop = "function process(log) end"; + + /* records continuing a group from a previous append, then their end */ + run_split_layout(noop, "drop", "r0 r1 E Sg r2 E", "r0 r1 E Sg r2 E"); + /* a group left open for the next append */ + run_split_layout(noop, "drop", "Sg r0 r1", "Sg r0 r1"); + run_split_layout(noop, "drop", "r0 E Sg r1 E Sh r2", "r0 E Sg r1 E Sh r2"); + /* an open group whose records are all dropped still opens for the next append */ + run_split_layout("local fb=require('fluentbit'); " + "function process(log) if log.body:get_number('index') > 0 " + "then return fb.DROP end end", + "drop", "r0 E Sg r1 r2", "r0 E Sg"); + /* only the first stray end before any start belongs to a previous append */ + run_split_layout(noop, "drop", "r0 E E Sg r1 E E r2", "r0 E Sg r1 E r2"); + /* + * a nested start begins a new flat context; nested open groups are not a + * single group continued by the next append, so the open one is dropped + */ + run_split_layout(noop, "drop", "Sg r0 Sh r1", "Sg r0 E r1"); +} + +static void open_group_edits_reopen_the_source_context(void) +{ + /* the last record moves to an edited context, the source group stays open */ + run_split_layout("function process(log) if log.body:get_number('index') == 1 then " + "log.context.attributes:set('producer', 'edited') end end", + "drop", "Sg r0 r1", "Sg r0 E Sedited r1 E Sg"); +} + +static void continuation_records_cannot_set_a_context(void) +{ + const char *code = "function process(log) if log.body:get_number('index') == 0 then " + "log.context.attributes:set('producer', 'new') end end"; + + run_split_layout(code, "keep", "r0 r1 E Sg r2 E", "r0 r1 E Sg r2 E"); + run_split_layout(code, "drop", "r0 r1 E Sg r2 E", "r1 E Sg r2 E"); +} + +static void duplicate_context_keys_keep_edits(void) +{ + /* + * 'duplicate' holds 'one' and 'two': setting it to the first value must + * still remove the second copy instead of looking unchanged. + */ + run_policy("function process(log) log.context.attributes:set('duplicate', 'one'); " + "assert(log.context.attributes:get_string('duplicate') == 'one') end", + 11, "drop", 6, 2, 0, NULL, NULL); +} + +static void json_large_objects_parse_in_linear_time(void) +{ + /* 60000 distinct keys under the default instruction budget, then a duplicate */ + run_policy("local fb=require('fluentbit'); local parts={}; " + "for i=1,60000 do parts[i]=string.format('\"k%06d\":0', i) end; " + "local text='{'..table.concat(parts, ',')..'}'; " + "local dup='{'..table.concat(parts, ',')..',\"k000001\":1}'; " + "function process(log) " + "assert(fb.parse_json(text):len() == 60000); " + "assert(not pcall(fb.parse_json, dup)) end", + 0, "drop", 6, 0, 0, "memory_limit", "128M"); +} + +TEST_LIST = { + {"empty_metadata_cow_isolation", empty_metadata_is_materialized_independently}, + {"edited_record_one_policy_tree", edited_record_needs_one_policy_tree}, + {"native_result_arena_snapshot_lifetime", native_result_arena_does_not_own_lua_snapshots}, + {"scalar_getters_live_paths", scalar_getters_preserve_types_and_live_paths}, + {"copied_values_binary_source_lifetime", copied_values_preserve_binary_data_and_source_lifetime}, + {"plain_log_root_validation", plain_log_fast_path_preserves_root_validation}, + {"drop_spans_removes_empty_groups", dropped_spans_remove_empty_resource_groups}, + {"array_insert_inherited_invalid_value", array_insert_preserves_inherited_invalid_value}, + {"nested_stray_group_markers", nested_and_stray_group_markers_preserve_records}, + {"invalid_mutations_accounting", invalid_mutations_do_not_accumulate_charges}, + {"native_nan_equality", native_nan_equality_preserves_unchanged_values}, + {"mixed_integer_addition_exact", mixed_integer_addition_preserves_exact_values}, + {"log_edit_empty_placeholders", log_edits_do_not_export_empty_read_placeholders}, + {"minimum_memory_all_signals", minimum_memory_initializes_all_signals}, + {"print_host_io_unavailable", print_is_not_a_host_io_escape}, + {"json_duplicate_detection_byte_exact", json_duplicate_detection_is_byte_exact}, + {"duplicate_context_metadata_groups", duplicate_context_metadata_does_not_split_groups}, + {"remove_empty_context_field", removing_empty_context_field_is_preserved}, + {"inherited_duplicate_invalid_utf8", inherited_duplicate_invalid_utf8_is_preserved}, + {"json_source_limit_large_scalar", json_source_limit_allows_large_scalar}, + {"documented_depth_enforced", documented_depth_is_enforced}, + {"replaced_redacted_values_accounting", replaced_and_redacted_values_do_not_accumulate_charges}, + {"memory_exhaustion_callback_recovery", memory_exhaustion_does_not_poison_later_callbacks}, + {"policies_cannot_stop_automatic_gc", policies_cannot_stop_automatic_gc}, + {"event_metatable_cannot_bypass_validation", event_metatable_cannot_bypass_validation}, + {"untouched_otel_fields_are_preserved", untouched_otel_fields_are_preserved}, + {"schema_promotion_is_record_local", schema_promotion_is_record_local}, + {"reserved_timestamps_are_record_errors", reserved_timestamps_are_record_errors}, + {"small_record_batch_uses_wire_budget", small_record_batch_uses_wire_budget}, + {"trusted_policy_jit_compiles", trusted_policy_jit_compiles}, + {"native_histogram_buckets_are_preserved", native_histogram_buckets_are_preserved}, + {"native_metric_scalar_preservation", native_metric_scalar_preservation}, + {"kept_native_family_no_merge", kept_native_family_does_not_merge_into_outputs}, + {"generic_context_isolation", generic_context_isolation}, + {"generic_read_does_not_split", generic_read_does_not_split}, + {"plain_logs_do_not_gain_context", plain_logs_do_not_gain_context}, + {"conditional_drop_preserves_groups", conditional_drop_preserves_groups}, + {"drop_all_removes_empty_groups", drop_all_removes_empty_groups}, + {"duplicate_redaction_canonicalizes", duplicate_redaction_canonicalizes}, + {"errors_restore_body_and_context", errors_restore_body_and_context}, + {"instruction_exhaustion_drops", instruction_exhaustion_drops}, + {"caught_budget_errors_terminate", caught_budget_errors_terminate}, + {"vm_memory_exhaustion_drops", vm_memory_exhaustion_drops}, + {"native_memory_exhaustion_drops", native_memory_exhaustion_drops}, + {"strict_parsing_and_time", strict_parsing_and_time}, + {"output_growth_fails_closed", output_growth_fails_closed}, + {"output_growth_keep_restores_whole_chunk", output_growth_keep_restores_whole_chunk}, + {"initialization_failures", initialization_failures}, + {"unclosed_group_preserves_records", unclosed_group_preserves_records}, + {"exact_opaque_context_ids", exact_opaque_context_ids}, + {"context_replacement_during_collection", context_replacement_during_collection}, + {"deep_mutable_views", deep_mutable_views}, + {"garbage_collector_is_private", garbage_collector_is_private}, + {"groups_split_across_appends", groups_split_across_appends_are_preserved}, + {"open_group_edits_reopen_source", open_group_edits_reopen_the_source_context}, + {"continuation_records_no_context", continuation_records_cannot_set_a_context}, + {"duplicate_context_keys_keep_edits", duplicate_context_keys_keep_edits}, + {"json_large_objects_linear", json_large_objects_parse_in_linear_time}, + {NULL, NULL} +}; diff --git a/tests/internal/mp.c b/tests/internal/mp.c index 66d9ab3f14f..bf166758c43 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -5,7 +5,10 @@ #include #include #include +#include #include +#include +#include #include #include @@ -742,7 +745,413 @@ static void test_preflight_formats(void) flb_mp_preflight_destroy(scan); } +/* Compare optimized counts with the authoritative decoder, including every + * truncated prefix and byte corruption of grouped and ordinary records. */ +static int reference_log_count(const char *data, size_t size) +{ + struct flb_log_event_decoder decoder; + struct flb_log_event event; + int count = 0; + + if (flb_log_event_decoder_init(&decoder, (char *) data, size) != 0) { + return 0; + } + while (flb_log_event_decoder_next(&decoder, &event) == 0) { + count++; + } + flb_log_event_decoder_destroy(&decoder); + return count; +} + +/* a processor that drops every record leaves an empty, NULL buffer */ +static void test_log_helpers_accept_empty_buffers(void) +{ + int count; + int records; + char *out_buf; + size_t out_size; + size_t window_end; + + count = -1; + TEST_CHECK(flb_mp_scan_log_records(NULL, 0, &count) == 0 && count == 0); + TEST_CHECK(flb_mp_scan_log_records(NULL, 1, &count) == -1); + TEST_CHECK(flb_mp_count_log_records(NULL, 0) == 0); + TEST_CHECK(flb_mp_log_group_boundaries(NULL, 0) == 0); + TEST_CHECK(flb_mp_log_group_starts(NULL, 0) == 0); + TEST_CHECK(flb_mp_log_window_next(NULL, 0, 0, 65536, 1024, + &window_end, &records) == -1); + + out_buf = (char *) 1; + out_size = 1; + TEST_CHECK(flb_mp_normalize_log_buffer_groups_msgpack_ex(NULL, 0, &out_buf, &out_size, + FLB_MP_GROUPS_KEEP_LEADING_END | + FLB_MP_GROUPS_KEEP_TRAILING_OPEN) == 0); + TEST_CHECK(out_buf == NULL && out_size == 0); +} + +static void test_log_scan_matches_decoder(void) +{ + struct flb_log_event_encoder *encoder; + struct flb_time timestamp; + size_t length; + size_t index; + int ret; + int count; + int record; + char saved; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (record = 0; record < 4; record++) { + if (record == 1) { + TEST_ASSERT(flb_log_event_encoder_group_init(encoder) == 0); + TEST_ASSERT(flb_log_event_encoder_group_header_end(encoder) == 0); + } + TEST_ASSERT(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(×tamp, 1700000000 + record, record); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(encoder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("key"), FLB_LOG_EVENT_CSTRING_VALUE("value")); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(encoder) == 0); + if (record == 2) { + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + } + } + TEST_CHECK(flb_mp_scan_log_records(encoder->output_buffer, encoder->output_length, + &count) == 0); + TEST_CHECK(count == 4); + for (length = 0; length <= encoder->output_length; length++) { + TEST_CHECK(flb_mp_count_log_records(encoder->output_buffer, length) == + reference_log_count(encoder->output_buffer, length)); + } + for (index = 0; index < encoder->output_length; index++) { + saved = encoder->output_buffer[index]; + encoder->output_buffer[index] = (char) 0xc1; + TEST_CHECK(flb_mp_count_log_records(encoder->output_buffer, encoder->output_length) == + reference_log_count(encoder->output_buffer, encoder->output_length)); + encoder->output_buffer[index] = saved; + + /* + * Leave the counts of 16 and 32 bit array and map headers intact: a + * corrupted count makes the msgpack-c fallback request gigabytes. + */ + if ((unsigned char) saved == 0xdc || (unsigned char) saved == 0xde) { + index += 2; + } + else if ((unsigned char) saved == 0xdd || (unsigned char) saved == 0xdf) { + index += 4; + } + } + flb_log_event_encoder_destroy(encoder); +} + +static void append_window_test_record(struct flb_log_event_encoder *encoder, int seq) +{ + int ret; + struct flb_time timestamp; + + TEST_ASSERT(flb_log_event_encoder_begin_record(encoder) == 0); + flb_time_set(×tamp, 1700000000 + seq, 0); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(encoder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(encoder, + FLB_LOG_EVENT_CSTRING_VALUE("seq"), FLB_LOG_EVENT_INT64_VALUE(seq)); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(encoder) == 0); +} + +static void test_log_window_next(void) +{ + struct flb_log_event_encoder *encoder; + size_t offset; + size_t end; + size_t group_start; + size_t group_end; + int records; + int total; + int windows; + int seq; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + + /* 10 plain records, a group of 6 records, 4 plain records */ + for (seq = 0; seq < 10; seq++) { + append_window_test_record(encoder, seq); + } + group_start = encoder->output_length; + TEST_ASSERT(flb_log_event_encoder_group_init(encoder) == 0); + TEST_ASSERT(flb_log_event_encoder_group_header_end(encoder) == 0); + for (; seq < 16; seq++) { + append_window_test_record(encoder, seq); + } + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + group_end = encoder->output_length; + for (; seq < 20; seq++) { + append_window_test_record(encoder, seq); + } + + /* record limited windows cover the buffer without splitting the group */ + offset = 0; + total = 0; + windows = 0; + while (offset < encoder->output_length) { + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, + encoder->output_length, offset, + SIZE_MAX, 4, &end, &records) == 0); + TEST_CHECK(end > offset && end <= encoder->output_length); + TEST_CHECK(!(offset < group_start && end > group_start && end < group_end)); + TEST_CHECK(!(offset > group_start && offset < group_end)); + TEST_CHECK(flb_mp_count_log_records(encoder->output_buffer + offset, + end - offset) == records); + if (end <= group_start || offset >= group_end) { + TEST_CHECK(records <= 4); + } + total += records; + windows++; + offset = end; + } + TEST_CHECK(offset == encoder->output_length); + TEST_CHECK(total == 20); + /* 4 + 4 + (2 plain and the whole group of 6) + 4 */ + TEST_CHECK(windows == 4); + + /* a byte limit of one closes after every event outside the group */ + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + 0, 1, INT_MAX, &end, &records) == 0); + TEST_CHECK(records == 1); + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + group_start, 1, INT_MAX, &end, &records) == 0); + TEST_CHECK(end == group_end && records == 6); + TEST_CHECK(flb_mp_log_group_starts(encoder->output_buffer + group_start, + group_end - group_start) == 1); + + /* a single window for the whole buffer */ + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + 0, SIZE_MAX, INT_MAX, &end, &records) == 0); + TEST_CHECK(end == encoder->output_length && records == 20); + + /* invalid input and offsets */ + TEST_CHECK(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + encoder->output_length, 1, 1, &end, &records) == -1); + TEST_CHECK(flb_mp_log_window_next(NULL, 10, 0, 1, 1, &end, &records) == -1); + TEST_CHECK(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length - 1, + 0, SIZE_MAX, INT_MAX, &end, &records) == -1); + TEST_CHECK(flb_mp_log_window_next("\x92\x01", 2, 0, 1, 1, &end, &records) == -1); + + flb_log_event_encoder_destroy(encoder); + + /* a full window keeps a following end marker instead of starting with it */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (seq = 0; seq < 4; seq++) { + append_window_test_record(encoder, seq); + } + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + group_end = encoder->output_length; + TEST_ASSERT(flb_log_event_encoder_group_init(encoder) == 0); + TEST_ASSERT(flb_log_event_encoder_group_header_end(encoder) == 0); + append_window_test_record(encoder, seq); + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + 0, SIZE_MAX, 4, &end, &records) == 0); + TEST_CHECK(end == group_end && records == 4); + TEST_ASSERT(flb_mp_log_window_next(encoder->output_buffer, encoder->output_length, + end, SIZE_MAX, 4, &end, &records) == 0); + TEST_CHECK(end == encoder->output_length && records == 1); + + flb_log_event_encoder_destroy(encoder); +} + +/* layout tokens: r record, S group start, E group end */ +static void encode_group_layout(struct flb_log_event_encoder *encoder, const char *layout) +{ + int length; + char token[16]; + const char *cursor; + + for (cursor = layout; sscanf(cursor, "%15s%n", token, &length) == 1; cursor += length) { + if (token[0] == 'r') { + append_window_test_record(encoder, atoi(token + 1)); + } + else if (token[0] == 'S') { + TEST_CHECK(flb_log_event_encoder_group_init(encoder) == 0); + TEST_CHECK(flb_log_event_encoder_group_header_end(encoder) == 0); + } + else { + TEST_CHECK(flb_log_event_encoder_group_end(encoder) == 0); + } + } +} + +static void describe_group_layout(char *buf, size_t size, char *out, size_t out_size) +{ + int type; + char token[16]; + msgpack_object *seq; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + + out[0] = '\0'; + if (buf == NULL || size == 0) { + return; + } + + TEST_ASSERT(flb_log_event_decoder_init(&decoder, buf, size) == 0); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type == FLB_LOG_EVENT_GROUP_START) { + snprintf(token, sizeof(token), "S "); + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + snprintf(token, sizeof(token), "E "); + } + else { + seq = &event.body->via.map.ptr[0].val; + snprintf(token, sizeof(token), "r%d ", (int) seq->via.i64); + } + strncat(out, token, out_size - strlen(out) - 1); + } + flb_log_event_decoder_destroy(&decoder); + if (strlen(out) > 0) { + out[strlen(out) - 1] = '\0'; + } +} + +static void check_normalized_layout(char *output, size_t output_size, + const char *layout, const char *expected, + const char *mode) +{ + char result[128]; + + describe_group_layout(output, output ? output_size : 0, result, sizeof(result)); + TEST_CHECK(strcmp(result, expected) == 0); + TEST_MSG("%s: '%s' produced '%s', expected '%s'", mode, layout, result, expected); +} + +/* + * 'strict' is the default normalization. 'kept' keeps the boundary markers + * the input already had, as processor chains do for groups split across + * appends. + */ +static void check_group_normalization(const char *layout, const char *strict, + const char *kept) +{ + struct flb_log_event_encoder *encoder; + struct flb_log_event_encoder *copy; + struct flb_mp_chunk_cobj *chunk; + struct flb_log_event_decoder *decoder; + struct flb_mp_chunk_record *record; + char *output; + size_t output_size; + int flags; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + encode_group_layout(encoder, layout); + flags = flb_mp_log_group_boundaries(encoder->output_buffer, encoder->output_length); + + output = NULL; + TEST_CHECK(flb_mp_normalize_log_buffer_groups_msgpack(encoder->output_buffer, + encoder->output_length, + &output, &output_size) == 0); + check_normalized_layout(output, output_size, layout, strict, "msgpack strict"); + flb_free(output); + + output = NULL; + TEST_CHECK(flb_mp_normalize_log_buffer_groups(encoder->output_buffer, + encoder->output_length, + &output, &output_size) == 0); + check_normalized_layout(output, output_size, layout, strict, "chunk strict"); + flb_free(output); + + output = NULL; + TEST_CHECK(flb_mp_normalize_log_buffer_groups_msgpack_ex(encoder->output_buffer, + encoder->output_length, + &output, &output_size, + flags) == 0); + check_normalized_layout(output, output_size, layout, kept, "msgpack kept"); + flb_free(output); + + /* the chunk normalizer with the same flags */ + decoder = flb_log_event_decoder_create(encoder->output_buffer, encoder->output_length); + TEST_ASSERT(decoder != NULL); + flb_log_event_decoder_read_groups(decoder, FLB_TRUE); + copy = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(copy != NULL); + chunk = flb_mp_chunk_cobj_create(copy, decoder); + TEST_ASSERT(chunk != NULL); + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + } + TEST_CHECK(flb_mp_chunk_cobj_normalize_groups_ex(chunk, flags) == 0); + output = NULL; + output_size = 0; + if (cfl_list_size(&chunk->records) > 0) { + TEST_CHECK(flb_mp_chunk_cobj_encode(chunk, &output, &output_size) == 0); + } + check_normalized_layout(output, output_size, layout, kept, "chunk kept"); + flb_free(output); + flb_mp_chunk_cobj_destroy(chunk); + flb_log_event_encoder_destroy(copy); + flb_log_event_decoder_destroy(decoder); + + flb_log_event_encoder_destroy(encoder); +} + +static void test_group_normalization_keeps_append_boundaries(void) +{ + /* a group continued in the next append keeps its start when kept */ + check_group_normalization("S r0 r1", "r0 r1", "S r0 r1"); + check_group_normalization("S", "", "S"); + /* records closing a group opened by a previous append keep their end */ + check_group_normalization("r0 r1 E", "r0 r1", "r0 r1 E"); + check_group_normalization("r0 E S r1 E S r2", "r0 S r1 E r2", "r0 E S r1 E S r2"); + /* other stray ends, empty groups and nested open groups are always removed */ + check_group_normalization("r0 E E S r1 E E r2", "r0 S r1 E r2", "r0 E S r1 E r2"); + check_group_normalization("S E r0", "r0", "r0"); + check_group_normalization("S S r0", "r0", "r0"); + check_group_normalization("S r0 E S E r1", "S r0 E r1", "S r0 E r1"); + check_group_normalization("S r0 E r1", "S r0 E r1", "S r0 E r1"); +} + +static void test_group_boundaries_follow_the_input(void) +{ + struct flb_log_event_encoder *encoder; + int flags; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + encode_group_layout(encoder, "r0 E S r1 E S r2"); + flags = flb_mp_log_group_boundaries(encoder->output_buffer, encoder->output_length); + TEST_CHECK(flags == (FLB_MP_GROUPS_KEEP_LEADING_END | FLB_MP_GROUPS_KEEP_TRAILING_OPEN)); + flb_log_event_encoder_destroy(encoder); + + /* a balanced input keeps nothing, whatever a processor removes later */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + encode_group_layout(encoder, "S r0 E r1"); + TEST_CHECK(flb_mp_log_group_boundaries(encoder->output_buffer, encoder->output_length) == 0); + flb_log_event_encoder_destroy(encoder); + + /* nested open groups are not a single continued group */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + encode_group_layout(encoder, "S S r0"); + TEST_CHECK(flb_mp_log_group_boundaries(encoder->output_buffer, encoder->output_length) == 0); + flb_log_event_encoder_destroy(encoder); + + TEST_CHECK(flb_mp_log_group_boundaries(NULL, 0) == 0); + TEST_CHECK(flb_mp_log_group_boundaries("\x92\x01", 2) == 0); +} + TEST_LIST = { + {"log_helpers_accept_empty_buffers", test_log_helpers_accept_empty_buffers}, + {"log_scan_matches_decoder", test_log_scan_matches_decoder}, + {"log_window_next", test_log_window_next}, + {"group_normalization_append_boundaries", test_group_normalization_keeps_append_boundaries}, + {"group_boundaries_follow_the_input", test_group_boundaries_follow_the_input}, {"preflight", test_preflight}, {"preflight_formats", test_preflight_formats}, {"validate_trace_chunk" , test_validate_trace_chunk}, diff --git a/tests/internal/mp_chunk_cobj.c b/tests/internal/mp_chunk_cobj.c index e10be583482..eb7e93f1242 100644 --- a/tests/internal/mp_chunk_cobj.c +++ b/tests/internal/mp_chunk_cobj.c @@ -26,6 +26,7 @@ #include #include +#include #include @@ -553,7 +554,195 @@ void record_next_condition_filter() } +static void bounded_record_pool_preserves_moved_records(void) +{ + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder = {0}; + struct flb_mp_chunk_cobj *source; + struct flb_mp_chunk_cobj *destination; + struct flb_mp_chunk_record *records[1024]; + int moved[] = {0, 511, 512, 1023}; + size_t index; + size_t pooled = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + source = flb_mp_chunk_cobj_create(encoder, &decoder); + destination = flb_mp_chunk_cobj_create(encoder, &decoder); + TEST_ASSERT(source != NULL && destination != NULL); + source->record_pool_enabled = FLB_TRUE; + for (index = 0; index < 1024; index++) { + records[index] = flb_mp_chunk_record_create(source); + TEST_ASSERT(records[index] != NULL); + TEST_CHECK(records[index]->cobj_record == NULL); + TEST_CHECK(records[index]->owns_group_attributes == FLB_FALSE); + pooled += records[index]->storage_block != NULL; + records[index]->modified = index; + cfl_list_add(&records[index]->_head, &source->records); + } + TEST_CHECK(pooled == 512); + TEST_CHECK(source->record_pool_blocks == 8); + for (index = 0; index < sizeof(moved) / sizeof(moved[0]); index++) { + cfl_list_del(&records[moved[index]]->_head); + cfl_list_add(&records[moved[index]]->_head, &destination->records); + } + TEST_CHECK(flb_mp_chunk_cobj_destroy(source) == 0); + for (index = 0; index < sizeof(moved) / sizeof(moved[0]); index++) { + TEST_CHECK(records[moved[index]]->modified == moved[index]); + } + TEST_CHECK(flb_mp_chunk_cobj_destroy(destination) == 0); + flb_log_event_encoder_destroy(encoder); +} + +static void scratch_encode_preserves_field_boundaries(void) +{ + struct flb_log_event_encoder *builder; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_log_event_decoder verify; + struct flb_log_event event; + struct flb_mp_chunk_cobj *chunk; + struct flb_mp_chunk_record *record; + struct flb_time timestamp; + char payload[9000]; + size_t lengths[] = {0, 1, sizeof(payload), 2}; + size_t length; + size_t index; + char *output; + size_t output_size; + int ret; + + memset(payload, 0xff, sizeof(payload)); + payload[1] = '\0'; + builder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(builder != NULL && encoder != NULL); + flb_time_set(×tamp, 1, 0); + for (index = 0; index < 16; index++) { + length = lengths[index % 4]; + TEST_ASSERT(flb_log_event_encoder_begin_record(builder) == 0); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(builder, ×tamp) == 0); + if (index % 2) { + ret = flb_log_event_encoder_append_metadata_values(builder, + FLB_LOG_EVENT_CSTRING_VALUE("text"), FLB_LOG_EVENT_STRING_VALUE(payload, length)); + TEST_ASSERT(ret == 0); + } + ret = flb_log_event_encoder_append_body_values(builder, + FLB_LOG_EVENT_CSTRING_VALUE("binary"), FLB_LOG_EVENT_BINARY_VALUE(payload, length)); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(builder) == 0); + } + TEST_ASSERT(flb_log_event_decoder_init(&decoder, builder->output_buffer, + builder->output_length) == 0); + chunk = flb_mp_chunk_cobj_create(encoder, &decoder); + TEST_ASSERT(chunk != NULL); + flb_mp_chunk_cobj_value_pool_enable(chunk); + TEST_ASSERT(chunk->value_pool != NULL); + while ((ret = flb_mp_chunk_cobj_record_next(chunk, &record)) == FLB_MP_CHUNK_RECORD_OK) { + TEST_CHECK(record->cobj_record->variant->arena == chunk->value_pool); + } + TEST_ASSERT(ret == FLB_MP_CHUNK_RECORD_EOF); + TEST_ASSERT(flb_mp_chunk_cobj_encode(chunk, &output, &output_size) == 0); + TEST_ASSERT(flb_log_event_decoder_init(&verify, output, output_size) == 0); + index = 0; + while (flb_log_event_decoder_next(&verify, &event) == FLB_EVENT_DECODER_SUCCESS) { + length = lengths[index % 4]; + TEST_ASSERT(event.body->type == MSGPACK_OBJECT_MAP && event.body->via.map.size == 1); + TEST_ASSERT(event.body->via.map.ptr[0].val.type == MSGPACK_OBJECT_BIN); + TEST_CHECK(event.body->via.map.ptr[0].val.via.bin.size == length); + TEST_CHECK(memcmp(event.body->via.map.ptr[0].val.via.bin.ptr, payload, length) == 0); + TEST_ASSERT(event.metadata->type == MSGPACK_OBJECT_MAP); + TEST_CHECK(event.metadata->via.map.size == index % 2); + if (index % 2) { + TEST_ASSERT(event.metadata->via.map.ptr[0].val.type == MSGPACK_OBJECT_STR); + TEST_CHECK(event.metadata->via.map.ptr[0].val.via.str.size == length); + TEST_CHECK(memcmp(event.metadata->via.map.ptr[0].val.via.str.ptr, payload, length) == 0); + } + index++; + } + TEST_CHECK(index == 16); + flb_log_event_decoder_destroy(&verify); + flb_mp_chunk_cobj_destroy(chunk); + flb_log_event_decoder_destroy(&decoder); + flb_log_event_encoder_destroy(builder); + flb_log_event_encoder_destroy(encoder); + flb_free(output); +} + +static void scratch_encode_failure_releases_storage(void) +{ + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder = {0}; + struct flb_mp_chunk_cobj *chunk; + struct flb_mp_chunk_record *record; + char *output = NULL; + size_t output_size = 0; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + chunk = flb_mp_chunk_cobj_create(encoder, &decoder); + TEST_ASSERT(chunk != NULL); + record = flb_mp_chunk_record_create(chunk); + TEST_ASSERT(record != NULL); + record->cobj_record = cfl_object_create(); + TEST_ASSERT(record->cobj_record != NULL); + cfl_list_add(&record->_head, &chunk->records); + TEST_CHECK(flb_mp_chunk_cobj_encode(chunk, &output, &output_size) == -1); + TEST_CHECK(output == NULL && output_size == 0); + flb_mp_chunk_cobj_destroy(chunk); + flb_log_event_encoder_destroy(encoder); +} + +static void exhausted_value_pool_falls_back_to_heap(void) +{ + struct flb_log_event_encoder *builder; + struct flb_log_event_encoder *encoder; + struct flb_log_event_decoder decoder; + struct flb_mp_chunk_cobj *chunk; + struct flb_mp_chunk_record *record; + struct flb_time timestamp; + size_t blocks = 0; + int ret; + + builder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(builder != NULL && encoder != NULL); + flb_time_set(×tamp, 1, 0); + TEST_ASSERT(flb_log_event_encoder_begin_record(builder) == 0); + TEST_ASSERT(flb_log_event_encoder_set_timestamp(builder, ×tamp) == 0); + ret = flb_log_event_encoder_append_body_values(builder, + FLB_LOG_EVENT_CSTRING_VALUE("message"), FLB_LOG_EVENT_CSTRING_VALUE("preserved")); + TEST_ASSERT(ret == 0); + TEST_ASSERT(flb_log_event_encoder_commit_record(builder) == 0); + TEST_ASSERT(flb_log_event_decoder_init(&decoder, builder->output_buffer, + builder->output_length) == 0); + chunk = flb_mp_chunk_cobj_create(encoder, &decoder); + TEST_ASSERT(chunk != NULL); + flb_mp_chunk_cobj_value_pool_enable(chunk); + TEST_ASSERT(chunk->value_pool != NULL); + TEST_CHECK(flb_mp_chunk_cobj_value_pool_get(chunk, SIZE_MAX) == NULL); + /* Trees already in the pool can grow past the budget; new trees cannot */ + while (blocks < 80 && cfl_arena_malloc(chunk->value_pool, 16384) != NULL) { + blocks++; + } + TEST_CHECK(blocks == 80); + TEST_CHECK(chunk->value_pool_bytes > 1024 * 1024); + TEST_CHECK(flb_mp_chunk_cobj_value_pool_get(chunk, 512) == NULL); + TEST_ASSERT(flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK); + TEST_CHECK(record->cobj_record->variant->arena == NULL); + TEST_CHECK(record->cobj_metadata->variant->arena == NULL); + TEST_CHECK(cfl_kvlist_contains(record->cobj_record->variant->data.as_kvlist, "message") == 1); + flb_mp_chunk_cobj_destroy(chunk); + flb_log_event_decoder_destroy(&decoder); + flb_log_event_encoder_destroy(builder); + flb_log_event_encoder_destroy(encoder); +} + TEST_LIST = { + { "exhausted_value_pool_heap_fallback", exhausted_value_pool_falls_back_to_heap }, + { "bounded_record_pool_moved_lifetime", bounded_record_pool_preserves_moved_records }, + { "scratch_encode_field_boundaries", scratch_encode_preserves_field_boundaries }, + { "scratch_encode_failure_cleanup", scratch_encode_failure_releases_storage }, { "decoder_groups_cobj", decoder_groups_cobj }, { "record_next_condition_filter", record_next_condition_filter }, { 0 } diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 0967329a861..128aea0e7d0 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -44,6 +44,21 @@ #include #include #include +#include +#include +#ifdef _WIN32 +#include +#define test_dup(fd) _dup(fd) +#define test_dup2(fd, target) _dup2(fd, target) +#define test_close(fd) _close(fd) +#define test_fileno(stream) _fileno(stream) +#else +#include +#define test_dup(fd) dup(fd) +#define test_dup2(fd, target) dup2(fd, target) +#define test_close(fd) close(fd) +#define test_fileno(stream) fileno(stream) +#endif #include #include @@ -1353,6 +1368,15 @@ static void run_metrics_case(msgpack_object *case_obj, const char *case_name) exp_histogram = cfl_list_entry(context->exp_histograms.next, struct cmt_exp_histogram, _head); metric = &exp_histogram->map->metric; + /* JSON decoding preserves previous windows; check the latest + * point for cases that supplied several unlabeled observations. */ + if (exp_histogram->map->label_count == 0 && + !cfl_list_is_empty(&exp_histogram->map->metrics)) { + metric = cfl_list_entry(exp_histogram->map->metrics.prev, + struct cmt_metric, _head); + TEST_CHECK(cmt_metric_get_timestamp(metric) >= + cmt_metric_get_timestamp(&exp_histogram->map->metric)); + } test_check_metric_description_unit(exp_histogram_obj, &exp_histogram->opts, exp_histogram->map); @@ -3532,8 +3556,468 @@ void test_opentelemetry_metrics_cutoff() cmt_destroy(cmt); } +static void *test_find_metric_family(struct cfl_list *families, size_t head_offset, + size_t opts_offset, const char *name) +{ + struct cfl_list *head; + struct cmt_opts *opts; + char *family; + + cfl_list_foreach(head, families) { + family = ((char *) head) - head_offset; + opts = (struct cmt_opts *) (family + opts_offset); + if (opts->fqname != NULL && strcmp(opts->fqname, name) == 0) { + return family; + } + } + + return NULL; +} + +#define TEST_FIND_FAMILY(list, type, name) \ + ((type *) test_find_metric_family(list, offsetof(type, _head), offsetof(type, opts), name)) + +static struct cmt_metric *test_single_dynamic_sample(struct cmt_map *map) +{ + if (map == NULL || map->metric_static_set || cfl_list_size(&map->metrics) != 1) { + return NULL; + } + + return cfl_list_entry_first(&map->metrics, struct cmt_metric, _head); +} + +static size_t test_kvlist_key_count(struct cfl_kvlist *list, const char *key) +{ + size_t count; + struct cfl_list *head; + struct cfl_kvpair *pair; + + count = 0; + cfl_list_foreach(head, &list->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (strcmp(pair->key, key) == 0) { + count++; + } + } + + return count; +} + +/* the typed OTLP metadata of a merged sample must describe the last point */ +static void test_check_last_point_metadata(struct cmt_metric *sample, uint64_t start_time) +{ + struct cfl_variant *value; + + TEST_CHECK(sample->otlp_metadata != NULL); + if (sample->otlp_metadata == NULL) { + return; + } + + TEST_CHECK(test_kvlist_key_count(sample->otlp_metadata, "attributes") == 1); + TEST_CHECK(test_kvlist_key_count(sample->otlp_metadata, "start_time_unix_nano") == 1); + + value = cfl_kvlist_fetch(sample->otlp_metadata, "start_time_unix_nano"); + TEST_CHECK(value != NULL && + (value->type == CFL_VARIANT_UINT || value->type == CFL_VARIANT_INT)); + if (value != NULL && value->type == CFL_VARIANT_UINT) { + TEST_CHECK(value->data.as_uint64 == start_time); + } +} + +/* Repeated series inside one OTLP JSON request update one sample, last wins. */ +void test_opentelemetry_metrics_json_merges_repeated_series() +{ + int ret; + int result; + char *payload; + char *msgpack_buffer; + size_t msgpack_size; + flb_sds_t otlp_json; + struct cfl_list contexts; + struct cmt *context; + struct cmt_gauge *gauge; + struct cmt_counter *counter; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *sample; + + payload = + "{\"resourceMetrics\":[{\"resource\":{},\"scopeMetrics\":[{\"scope\":{\"name\":\"s\"}," + "\"metrics\":[" + "{\"name\":\"g\",\"gauge\":{\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\",\"asDouble\":1.0}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\",\"asDouble\":2.0}" + "]}}," + "{\"name\":\"u\",\"gauge\":{\"dataPoints\":[" + "{\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\",\"asDouble\":1.0}," + "{\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\",\"asDouble\":2.0}" + "]}}," + "{\"name\":\"c\",\"sum\":{\"aggregationTemporality\":2,\"isMonotonic\":true," + "\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\",\"asInt\":\"10\"}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\",\"asInt\":\"20\"}" + "]}}," + "{\"name\":\"h\",\"histogram\":{\"aggregationTemporality\":2,\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\"," + "\"count\":\"3\",\"sum\":3.0,\"bucketCounts\":[\"1\",\"2\"],\"explicitBounds\":[1.0]}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\"," + "\"count\":\"7\",\"sum\":9.0,\"bucketCounts\":[\"3\",\"4\"],\"explicitBounds\":[1.0]}" + "]}}," + "{\"name\":\"sm\",\"summary\":{\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\"," + "\"count\":\"2\",\"sum\":4.0," + "\"quantileValues\":[{\"quantile\":0.5,\"value\":1.0}]}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\"," + "\"count\":\"5\",\"sum\":8.0," + "\"quantileValues\":[{\"quantile\":0.5,\"value\":6.0}]}" + "]}}," + "{\"name\":\"e\",\"exponentialHistogram\":{\"aggregationTemporality\":2," + "\"dataPoints\":[" + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"1\",\"timeUnixNano\":\"1700000001000000000\"," + "\"count\":\"1\",\"sum\":1.0,\"scale\":0,\"zeroCount\":\"0\"," + "\"positive\":{\"offset\":0,\"bucketCounts\":[\"1\"]}}," + "{\"attributes\":[{\"key\":\"host\",\"value\":{\"stringValue\":\"h1\"}}]," + "\"startTimeUnixNano\":\"2\",\"timeUnixNano\":\"1700000002000000000\"," + "\"count\":\"4\",\"sum\":6.0,\"scale\":0,\"zeroCount\":\"1\"," + "\"positive\":{\"offset\":1,\"bucketCounts\":[\"1\",\"2\"]}}" + "]}}" + "]}]}]}"; + + cfl_list_init(&contexts); + + ret = flb_opentelemetry_metrics_json_to_cmt(&contexts, payload, strlen(payload)); + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_list_size(&contexts) == 1); + if (ret != 0 || cfl_list_size(&contexts) != 1) { + destroy_metrics_context_list(&contexts); + return; + } + + context = cfl_list_entry_first(&contexts, struct cmt, _head); + + /* gauge with labels */ + gauge = TEST_FIND_FAMILY(&context->gauges, struct cmt_gauge, "g"); + TEST_CHECK(gauge != NULL); + sample = gauge != NULL ? test_single_dynamic_sample(gauge->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(cmt_metric_get_value(sample) == 2.0); + test_check_last_point_metadata(sample, 2); + } + + /* gauge without labels: both points land on the static sample */ + gauge = TEST_FIND_FAMILY(&context->gauges, struct cmt_gauge, "u"); + TEST_CHECK(gauge != NULL); + if (gauge != NULL) { + TEST_CHECK(gauge->map->metric_static_set == CMT_TRUE); + TEST_CHECK(cfl_list_size(&gauge->map->metrics) == 0); + TEST_CHECK(cmt_metric_get_value(&gauge->map->metric) == 2.0); + test_check_last_point_metadata(&gauge->map->metric, 2); + } + + /* sum */ + counter = TEST_FIND_FAMILY(&context->counters, struct cmt_counter, "c"); + TEST_CHECK(counter != NULL); + sample = counter != NULL ? test_single_dynamic_sample(counter->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(cmt_metric_get_value(sample) == 20.0); + test_check_last_point_metadata(sample, 2); + } + + /* histogram */ + histogram = TEST_FIND_FAMILY(&context->histograms, struct cmt_histogram, "h"); + TEST_CHECK(histogram != NULL); + sample = histogram != NULL ? test_single_dynamic_sample(histogram->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(cmt_metric_hist_get_count_value(sample) == 7); + TEST_CHECK(cmt_metric_hist_get_sum_value(sample) == 9.0); + TEST_CHECK(cmt_metric_hist_get_bucket_count(sample, 0) == 3); + TEST_CHECK(cmt_metric_hist_get_bucket_count(sample, 1) == 4); + test_check_last_point_metadata(sample, 2); + } + + /* summary */ + summary = TEST_FIND_FAMILY(&context->summaries, struct cmt_summary, "sm"); + TEST_CHECK(summary != NULL); + sample = summary != NULL ? test_single_dynamic_sample(summary->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(cmt_summary_get_count_value(sample) == 5); + TEST_CHECK(cmt_summary_get_sum_value(sample) == 8.0); + TEST_CHECK(cmt_summary_quantile_get_value(sample, 0) == 6.0); + test_check_last_point_metadata(sample, 2); + } + + /* exponential histogram */ + exp_histogram = TEST_FIND_FAMILY(&context->exp_histograms, struct cmt_exp_histogram, "e"); + TEST_CHECK(exp_histogram != NULL); + sample = exp_histogram != NULL ? test_single_dynamic_sample(exp_histogram->map) : NULL; + TEST_CHECK(sample != NULL); + if (sample != NULL) { + TEST_CHECK(sample->exp_hist_count == 4); + TEST_CHECK(sample->exp_hist_zero_count == 1); + TEST_CHECK(sample->exp_hist_positive_offset == 1); + TEST_CHECK(sample->exp_hist_positive_count == 2); + if (sample->exp_hist_positive_count == 2) { + TEST_CHECK(sample->exp_hist_positive_buckets[1] == 2); + } + test_check_last_point_metadata(sample, 2); + } + + /* the re-encoded payload carries a single data point per series */ + ret = cmt_encode_msgpack_create(context, &msgpack_buffer, &msgpack_size); + TEST_CHECK(ret == 0); + if (ret == 0) { + otlp_json = flb_opentelemetry_metrics_msgpack_to_otlp_json(msgpack_buffer, + msgpack_size, + &result); + TEST_CHECK(otlp_json != NULL); + if (otlp_json != NULL) { + TEST_CHECK(strstr(otlp_json, "\"asInt\":\"20\"") != NULL); + TEST_CHECK(strstr(otlp_json, "\"asInt\":\"10\"") == NULL); + TEST_CHECK(strstr(otlp_json, "\"startTimeUnixNano\":\"1\"") == NULL); + flb_sds_destroy(otlp_json); + } + flb_free(msgpack_buffer); + } + + destroy_metrics_context_list(&contexts); +} + +static size_t test_count_substring(const char *haystack, const char *needle) +{ + size_t count; + const char *cursor; + + count = 0; + cursor = haystack; + while ((cursor = strstr(cursor, needle)) != NULL) { + count++; + cursor += strlen(needle); + } + + return count; +} + +/* A list with many invalid attributes is reported once, healthy ones are kept. */ +void test_json_invalid_attributes_warn_once() +{ + int ret; + int type; + int saved_stderr; + int index; + char *buffer; + char log_buffer[4096]; + size_t size; + size_t offset; + size_t log_size; + FILE *capture; + flb_sds_t json; + msgpack_unpacked unpacked; + struct cfl_kvlist *attributes; + + json = flb_sds_create("["); + TEST_ASSERT(json != NULL); + for (index = 0; index < 50; index++) { + flb_sds_printf(&json, "{\"key\":\"b%d\",\"value\":{\"bytesValue\":\"!!!\"}},", index); + } + flb_sds_cat_safe(&json, "{\"key\":\"ok\",\"value\":{\"stringValue\":\"kept\"}}]", + sizeof("{\"key\":\"ok\",\"value\":{\"stringValue\":\"kept\"}}]") - 1); + + ret = flb_pack_json(json, flb_sds_len(json), &buffer, &size, &type, NULL); + TEST_ASSERT(ret == 0); + + offset = 0; + msgpack_unpacked_init(&unpacked); + TEST_ASSERT(msgpack_unpack_next(&unpacked, buffer, size, &offset)); + + attributes = cfl_kvlist_create(); + TEST_ASSERT(attributes != NULL); + + capture = tmpfile(); + TEST_ASSERT(capture != NULL); + + fflush(stderr); + saved_stderr = test_dup(test_fileno(stderr)); + TEST_ASSERT(saved_stderr >= 0); + test_dup2(test_fileno(capture), test_fileno(stderr)); + + ret = flb_otel_utils_clone_kvlist_from_otlp_json_array(attributes, &unpacked.data); + + fflush(stderr); + test_dup2(saved_stderr, test_fileno(stderr)); + test_close(saved_stderr); + + TEST_CHECK(ret == 0); + TEST_CHECK(cfl_kvlist_count(attributes) == 1); + TEST_CHECK(cfl_kvlist_fetch(attributes, "ok") != NULL); + + rewind(capture); + log_size = fread(log_buffer, 1, sizeof(log_buffer) - 1, capture); + log_buffer[log_size] = '\0'; + fclose(capture); + + TEST_CHECK(test_count_substring(log_buffer, "invalid OTLP JSON attribute") == 1); + TEST_CHECK(strstr(log_buffer, "ignoring 50 invalid") != NULL); + TEST_MSG("captured log: %s", log_buffer); + + cfl_kvlist_destroy(attributes); + msgpack_unpacked_destroy(&unpacked); + flb_free(buffer); + flb_sds_destroy(json); +} + +static int test_logs_json_to_otlp_json(char *payload, flb_sds_t *out) +{ + int ret; + int result; + struct flb_log_event_encoder encoder; + struct flb_opentelemetry_otlp_logs_options options; + + *out = NULL; + + ret = flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_DEFAULT); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + result = 0; + ret = flb_opentelemetry_logs_json_to_msgpack(&encoder, payload, strlen(payload), + "log", &result); + if (ret != 0 || result != 0) { + flb_log_event_encoder_destroy(&encoder); + return -1; + } + + memset(&options, 0, sizeof(options)); + options.logs_require_otel_metadata = FLB_TRUE; + options.logs_body_key = "log"; + + *out = flb_opentelemetry_logs_to_otlp_json(encoder.output_buffer, encoder.output_length, + &options, &result); + flb_log_event_encoder_destroy(&encoder); + + return *out != NULL ? 0 : -1; +} + +/* OTLP JSON bytesValue is base64 decoded and round trips as bytesValue. */ +void test_opentelemetry_logs_json_bytes_value() +{ + int ret; + char *payload; + flb_sds_t actual; + + payload = + "{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"rb\"," + "\"value\":{\"bytesValue\":\"AAE=\"}}]},\"scopeLogs\":[{\"scope\":{\"name\":\"s\"}," + "\"logRecords\":[{\"timeUnixNano\":\"1700000001000000000\"," + "\"body\":{\"bytesValue\":\"aGVsbG8=\"}," + "\"attributes\":[{\"key\":\"padded\",\"value\":{\"bytesValue\":\"AAH/\"}}," + "{\"key\":\"unpadded\",\"value\":{\"bytesValue\":\"aGVsbG8\"}}," + "{\"key\":\"urlsafe\",\"value\":{\"bytesValue\":\"-_8\"}}," + "{\"key\":\"empty\",\"value\":{\"bytesValue\":\"\"}}]}]}]}]}"; + + ret = test_logs_json_to_otlp_json(payload, &actual); + TEST_CHECK(ret == 0); + if (ret == 0) { + TEST_CHECK(strstr(actual, "\"body\":{\"bytesValue\":\"aGVsbG8=\"}") != NULL); + TEST_CHECK(strstr(actual, "{\"key\":\"rb\",\"value\":{\"bytesValue\":\"AAE=\"}}") != NULL); + TEST_CHECK(strstr(actual, + "{\"key\":\"padded\",\"value\":{\"bytesValue\":\"AAH/\"}}") != NULL); + TEST_CHECK(strstr(actual, "{\"key\":\"unpadded\"," + "\"value\":{\"bytesValue\":\"aGVsbG8=\"}}") != NULL); + TEST_CHECK(strstr(actual, + "{\"key\":\"urlsafe\",\"value\":{\"bytesValue\":\"+/8=\"}}") != NULL); + TEST_CHECK(strstr(actual, "stringValue") == NULL); + TEST_MSG("actual: %s", actual); + flb_sds_destroy(actual); + } + + /* invalid base64 is rejected like other invalid values */ + payload = + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[{" + "\"timeUnixNano\":\"1700000001000000000\",\"body\":{\"bytesValue\":\"!!!\"}}]}]}]}"; + ret = test_logs_json_to_otlp_json(payload, &actual); + TEST_CHECK(ret != 0); + if (actual != NULL) { + flb_sds_destroy(actual); + } + + payload = + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[{" + "\"timeUnixNano\":\"1700000001000000000\",\"body\":{\"stringValue\":\"x\"}," + "\"attributes\":[{\"key\":\"b\",\"value\":{\"bytesValue\":\"A\"}}]}]}]}]}"; + ret = test_logs_json_to_otlp_json(payload, &actual); + TEST_CHECK(ret != 0); + if (actual != NULL) { + flb_sds_destroy(actual); + } +} + /* Test list */ +static void test_json_anyvalue_signed_bytes_and_invalid_attributes(void) +{ + const char *json = "[{\"key\":\"negative\",\"value\":{\"intValue\":\"-9223372036854775808\"}}," + "{\"key\":\"positive\",\"value\":{\"intValue\":\"9223372036854775807\"}}," + "{\"key\":\"bytes\",\"value\":{\"bytesValue\":\"AAH/\"}}," + "{\"key\":\"overflow\",\"value\":{\"intValue\":\"18446744073709551615\"}}," + "{\"key\":\"invalid\",\"value\":{\"boolValue\":\"wrong\"}}," + "{\"key\":\"healthy\",\"value\":{\"stringValue\":\"kept\"}}]"; + char *buffer; + size_t size; + size_t offset = 0; + int type; + msgpack_unpacked unpacked; + struct cfl_kvlist *attributes; + struct cfl_variant *value; + const unsigned char bytes[] = {0, 1, 255}; + + TEST_ASSERT(flb_pack_json(json, strlen(json), &buffer, &size, &type, NULL) == 0); + msgpack_unpacked_init(&unpacked); + TEST_ASSERT(msgpack_unpack_next(&unpacked, buffer, size, &offset)); + attributes = cfl_kvlist_create(); + TEST_ASSERT(attributes != NULL); + TEST_CHECK(flb_otel_utils_clone_kvlist_from_otlp_json_array(attributes, &unpacked.data) == 0); + value = cfl_kvlist_fetch(attributes, "negative"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_INT); + if (value != NULL && value->type == CFL_VARIANT_INT) { + TEST_CHECK(value->data.as_int64 == INT64_MIN); + } + value = cfl_kvlist_fetch(attributes, "positive"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_INT); + if (value != NULL && value->type == CFL_VARIANT_INT) { + TEST_CHECK(value->data.as_int64 == INT64_MAX); + } + value = cfl_kvlist_fetch(attributes, "bytes"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_BYTES); + if (value != NULL && value->type == CFL_VARIANT_BYTES) { + TEST_CHECK(cfl_sds_len(value->data.as_bytes) == sizeof(bytes)); + TEST_CHECK(memcmp(value->data.as_bytes, bytes, sizeof(bytes)) == 0); + } + TEST_CHECK(cfl_kvlist_fetch(attributes, "overflow") == NULL); + TEST_CHECK(cfl_kvlist_fetch(attributes, "invalid") == NULL); + value = cfl_kvlist_fetch(attributes, "healthy"); + TEST_CHECK(value != NULL && value->type == CFL_VARIANT_STRING); + TEST_CHECK(cfl_kvlist_count(attributes) == 4); + cfl_kvlist_destroy(attributes); + msgpack_unpacked_destroy(&unpacked); + flb_free(buffer); +} + TEST_LIST = { + {"json_anyvalue_signed_bytes_invalid_attributes", test_json_anyvalue_signed_bytes_and_invalid_attributes}, { "hex_to_id", test_hex_to_id }, { "hex_to_id_error_cases", test_hex_to_id_error_cases }, { "convert_string_number_to_u64", test_convert_string_number_to_u64 }, @@ -3573,5 +4057,9 @@ TEST_LIST = { test_opentelemetry_metrics_otlp_proto_batches_empty_context }, { "opentelemetry_metrics_cutoff", test_opentelemetry_metrics_cutoff }, + { "opentelemetry_metrics_json_merges_repeated_series", + test_opentelemetry_metrics_json_merges_repeated_series }, + { "json_invalid_attributes_warn_once", test_json_invalid_attributes_warn_once }, + { "opentelemetry_logs_json_bytes_value", test_opentelemetry_logs_json_bytes_value }, { 0 } }; diff --git a/tests/internal/processor.c b/tests/internal/processor.c index e2ebcd7bf86..d77fba9d86f 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -18,6 +18,7 @@ */ #include +#include #include #include #include @@ -31,6 +32,8 @@ #include #include #include +#include +#include #include #include @@ -625,10 +628,829 @@ static void processor_metrics_counters() flb_sds_destroy(regex_prop_key); } +static int create_intermediate_metrics(struct flb_processor_instance *ins, + struct cmt *input, struct cmt **output, + const char *tag, int tag_len) +{ + *output = cmt_create(); + return *output == NULL ? -1 : FLB_PROCESSOR_SUCCESS; +} + +static int fail_metrics_processing(struct flb_processor_instance *ins, + struct cmt *input, struct cmt **output, + const char *tag, int tag_len) +{ + *output = NULL; + return -1; +} + +static void processor_metrics_failure_clears_output(void) +{ + struct flb_config *config; + struct flb_processor *proc; + struct cmt *input; + void *output; + size_t output_size = 123; + struct flb_processor_plugin first = { + .name = "test_metrics_copy", + .description = "create intermediate metrics", + .cb_process_metrics = create_intermediate_metrics + }; + struct flb_processor_plugin second = { + .name = "test_metrics_failure", + .description = "fail metrics processing", + .cb_process_metrics = fail_metrics_processing + }; + + flb_init_env(); + config = flb_config_init(); + TEST_ASSERT(config != NULL); + mk_list_add(&first._head, &config->processor_plugins); + mk_list_add(&second._head, &config->processor_plugins); + proc = flb_processor_create(config, "failure", NULL, 0); + TEST_ASSERT(proc != NULL); + TEST_ASSERT(flb_processor_unit_create(proc, FLB_PROCESSOR_METRICS, first.name) != NULL); + TEST_ASSERT(flb_processor_unit_create(proc, FLB_PROCESSOR_METRICS, second.name) != NULL); + TEST_ASSERT(flb_processor_init(proc) == 0); + input = cmt_create(); + TEST_ASSERT(input != NULL); + output = input; + TEST_CHECK(flb_processor_run(proc, 0, FLB_PROCESSOR_METRICS, "test", 4, + input, 0, &output, &output_size) == -1); + TEST_CHECK(output == NULL); + TEST_CHECK(output_size == 0); + /* The caller still owns the original input after a failed pipeline. */ + cmt_destroy(input); + flb_processor_destroy(proc); + mk_list_del(&first._head); + mk_list_del(&second._head); + flb_config_exit(config); +} + +/* records per group and around the groups of the windowed buffer */ +#define WINDOW_GROUPS 3 +#define WINDOW_GROUP_RECORDS 1500 +#define WINDOW_PLAIN_RECORDS 2000 +#define WINDOW_TOTAL_RECORDS (WINDOW_GROUPS * WINDOW_GROUP_RECORDS + \ + 2 * WINDOW_PLAIN_RECORDS) + +static int window_calls; +static int window_max_records; + +static int append_window_record(struct flb_log_event_encoder *encoder, int seq) +{ + int ret; + struct flb_time ts; + + ret = flb_log_event_encoder_begin_record(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + flb_time_set(&ts, 1700000000 + seq, 0); + ret = flb_log_event_encoder_set_timestamp(encoder, &ts); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + return -1; + } + + ret = flb_log_event_encoder_append_body_values( + encoder, + FLB_LOG_EVENT_CSTRING_VALUE("seq"), + FLB_LOG_EVENT_INT64_VALUE(seq), + FLB_LOG_EVENT_CSTRING_VALUE("message"), + FLB_LOG_EVENT_CSTRING_VALUE("windowed processing test record")); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_commit_record(encoder); + } + + return ret == FLB_EVENT_ENCODER_SUCCESS ? 0 : -1; +} + +/* plain records, groups larger than the window record limit, plain records */ +static int create_window_records(char **out_buf, size_t *out_size) +{ + int i; + int g; + int seq = 0; + int ret = 0; + char name[8]; + struct flb_log_event_encoder *encoder; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + for (i = 0; ret == 0 && i < WINDOW_PLAIN_RECORDS; i++) { + ret = append_window_record(encoder, seq++); + } + + for (g = 0; ret == 0 && g < WINDOW_GROUPS; g++) { + snprintf(name, sizeof(name), "g%d", g); + if (flb_log_event_encoder_group_init(encoder) != 0) { + ret = -1; + break; + } + + ret = flb_log_event_encoder_append_metadata_values( + encoder, + FLB_LOG_EVENT_CSTRING_VALUE("group"), + FLB_LOG_EVENT_CSTRING_VALUE(name)); + if (ret != FLB_EVENT_ENCODER_SUCCESS || + flb_log_event_encoder_group_header_end(encoder) != 0) { + ret = -1; + break; + } + + for (i = 0; ret == 0 && i < WINDOW_GROUP_RECORDS; i++) { + ret = append_window_record(encoder, seq++); + } + + if (ret == 0 && flb_log_event_encoder_group_end(encoder) != 0) { + ret = -1; + } + } + + for (i = 0; ret == 0 && i < WINDOW_PLAIN_RECORDS; i++) { + ret = append_window_record(encoder, seq++); + } + + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + *out_buf = flb_malloc(encoder->output_length); + if (*out_buf == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + memcpy(*out_buf, encoder->output_buffer, encoder->output_length); + *out_size = encoder->output_length; + flb_log_event_encoder_destroy(encoder); + + return 0; +} + +static int mark_window_records(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + int type; + int records = 0; + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_record *record; + + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + if (flb_log_event_decoder_get_record_type(&record->event, &type) != 0 || + type != FLB_LOG_EVENT_NORMAL) { + continue; + } + records++; + if (cfl_kvlist_insert_int64(record->cobj_record->variant->data.as_kvlist, + "x", 1) != 0) { + return FLB_PROCESSOR_FAILURE; + } + } + + window_calls++; + if (records > window_max_records) { + window_max_records = records; + } + + return FLB_PROCESSOR_SUCCESS; +} + +static int drop_window_records(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + int type; + struct cfl_list *head; + struct cfl_list *tmp; + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_record *record; + + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + } + chunk->record_pos = NULL; + + cfl_list_foreach_safe(head, tmp, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + if (flb_log_event_decoder_get_record_type(&record->event, &type) == 0 && + type == FLB_LOG_EVENT_NORMAL) { + flb_mp_chunk_cobj_record_destroy(chunk, record); + } + } + + window_calls++; + return FLB_PROCESSOR_SUCCESS; +} + +static struct flb_processor_plugin window_mark_plugin = { + .name = "test_window_mark", + .description = "windowed record marker", + .cb_process_logs = mark_window_records, + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS +}; + +static struct flb_processor_plugin single_mark_plugin = { + .name = "test_single_mark", + .description = "single chunk record marker", + .cb_process_logs = mark_window_records, + .flags = 0 +}; + +static struct flb_processor_plugin window_drop_plugin = { + .name = "test_window_drop", + .description = "windowed record dropper", + .cb_process_logs = drop_window_records, + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS +}; + +static struct flb_config *window_config_create(void) +{ + struct flb_config *config; + + flb_init_env(); + config = flb_config_init(); + if (config == NULL) { + return NULL; + } + mk_list_add(&window_mark_plugin._head, &config->processor_plugins); + mk_list_add(&single_mark_plugin._head, &config->processor_plugins); + mk_list_add(&window_drop_plugin._head, &config->processor_plugins); + + return config; +} + +static void window_config_destroy(struct flb_config *config) +{ + mk_list_del(&window_mark_plugin._head); + mk_list_del(&single_mark_plugin._head); + mk_list_del(&window_drop_plugin._head); + flb_config_exit(config); +} + +static struct flb_processor *window_processor_create(struct flb_config *config, + struct flb_input_instance *owner, + char *first, char *second) +{ + struct flb_processor *proc; + + proc = flb_processor_create(config, "window_test", NULL, 0); + if (proc == NULL) { + return NULL; + } + proc->data = owner; + proc->source_plugin_type = FLB_PLUGIN_INPUT; + + if (flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, first) == NULL || + (second && flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, second) == NULL) || + flb_processor_init(proc) != 0) { + flb_processor_destroy(proc); + return NULL; + } + + return proc; +} + +/* checks order, the processor mark and that every group stayed whole */ +static void check_window_output(char *buf, size_t size) +{ + int ret; + int type; + int seq = 0; + int groups = 0; + int in_group = FLB_FALSE; + int group_records = 0; + char name[8]; + msgpack_object *value; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + + ret = flb_log_event_decoder_init(&decoder, buf, size); + TEST_ASSERT(ret == FLB_EVENT_DECODER_SUCCESS); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + + if (type == FLB_LOG_EVENT_GROUP_START) { + TEST_CHECK(in_group == FLB_FALSE); + snprintf(name, sizeof(name), "g%d", groups); + value = &event.metadata->via.map.ptr[0].val; + TEST_CHECK(event.metadata->via.map.size == 1); + TEST_CHECK(value->type == MSGPACK_OBJECT_STR && + value->via.str.size == strlen(name) && + memcmp(value->via.str.ptr, name, strlen(name)) == 0); + in_group = FLB_TRUE; + group_records = 0; + continue; + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + TEST_CHECK(in_group == FLB_TRUE); + TEST_CHECK(group_records == WINDOW_GROUP_RECORDS); + in_group = FLB_FALSE; + groups++; + continue; + } + + if (in_group) { + group_records++; + } + TEST_CHECK(event.body->type == MSGPACK_OBJECT_MAP && + event.body->via.map.size == 3); + TEST_CHECK(event.body->via.map.ptr[0].val.via.i64 == seq); + TEST_CHECK(event.body->via.map.ptr[2].val.via.i64 == 1); + seq++; + } + + TEST_CHECK(seq == WINDOW_TOTAL_RECORDS); + TEST_CHECK(groups == WINDOW_GROUPS); + TEST_CHECK(in_group == FLB_FALSE); + flb_log_event_decoder_destroy(&decoder); +} + +static void processor_log_windows_match_single_chunk(void) +{ + int ret; + int records; + int single_calls; + char *input; + size_t input_size; + void *windowed; + size_t windowed_size; + void *single; + size_t single_size; + double items_in; + double items_out; + double invocations; + char stage_label[32]; + struct flb_config *config; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_input_instance owner; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + TEST_ASSERT(create_window_records(&input, &input_size) == 0); + TEST_CHECK(input_size > 4 * 65536); + + /* the single chunk reference */ + proc = window_processor_create(config, &owner, "test_single_mark", NULL); + TEST_ASSERT(proc != NULL); + window_calls = 0; + window_max_records = 0; + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &single, &single_size); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls == 1); + TEST_CHECK(window_max_records == WINDOW_TOTAL_RECORDS); + single_calls = window_calls; + flb_processor_destroy(proc); + + /* the windowed run must produce the same bytes */ + proc = window_processor_create(config, &owner, "test_window_mark", NULL); + TEST_ASSERT(proc != NULL); + window_calls = 0; + window_max_records = 0; + records = -1; + ret = flb_processor_run_with_count(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &windowed, &windowed_size, + -1, &records); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls > single_calls); + /* windows hold at most the record limit plus one whole group */ + TEST_CHECK(window_max_records <= 1024 + WINDOW_GROUP_RECORDS); + TEST_CHECK(records == WINDOW_TOTAL_RECORDS); + TEST_CHECK(windowed != NULL && windowed != input); + TEST_CHECK(windowed_size == single_size); + TEST_CHECK(windowed_size == single_size && + memcmp(windowed, single, single_size) == 0); + check_window_output(windowed, windowed_size); + + /* one invocation per buffer, with logical record counts */ + pu = mk_list_entry(proc->logs.next, struct flb_processor_unit, _head); + snprintf(stage_label, sizeof(stage_label), "%zu", pu->stage); + TEST_CHECK(get_counter_value_5(proc->cmt_invocations, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", + &invocations) == 0); + TEST_CHECK(get_counter_value_5(proc->cmt_items_in, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", + &items_in) == 0); + TEST_CHECK(get_counter_value_5(proc->cmt_items_out, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", + &items_out) == 0); + TEST_CHECK(invocations == 1.0); + TEST_CHECK(items_in == WINDOW_TOTAL_RECORDS); + TEST_CHECK(items_out == WINDOW_TOTAL_RECORDS); + flb_processor_destroy(proc); + + flb_free(windowed); + flb_free(single); + flb_free(input); + cmt_destroy(owner.cmt); + window_config_destroy(config); +} + +static void processor_log_windows_drop_all(void) +{ + int ret; + int records; + char *input; + size_t input_size; + void *output; + size_t output_size; + double items_drop; + char stage_label[32]; + struct flb_config *config; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_input_instance owner; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + TEST_ASSERT(create_window_records(&input, &input_size) == 0); + + proc = window_processor_create(config, &owner, "test_window_drop", NULL); + TEST_ASSERT(proc != NULL); + window_calls = 0; + records = -1; + output = input; + output_size = input_size; + ret = flb_processor_run_with_count(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &output, &output_size, + -1, &records); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls > 1); + TEST_CHECK(output == NULL); + TEST_CHECK(output_size == 0); + TEST_CHECK(records == 0); + + pu = mk_list_entry(proc->logs.next, struct flb_processor_unit, _head); + snprintf(stage_label, sizeof(stage_label), "%zu", pu->stage); + TEST_CHECK(get_counter_value_5(proc->cmt_items_drop, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", + &items_drop) == 0); + TEST_CHECK(items_drop == WINDOW_TOTAL_RECORDS); + + flb_processor_destroy(proc); + flb_free(input); + cmt_destroy(owner.cmt); + window_config_destroy(config); +} + +static void processor_log_windows_need_every_unit(void) +{ + int i; + int ret; + char *input; + size_t input_size; + void *output; + size_t output_size; + struct flb_config *config; + struct flb_processor *proc; + struct flb_input_instance owner; + struct flb_log_event_encoder *encoder; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + TEST_ASSERT(create_window_records(&input, &input_size) == 0); + + /* a unit without the capability keeps the whole run on one chunk */ + proc = window_processor_create(config, &owner, "test_window_mark", + "test_single_mark"); + TEST_ASSERT(proc != NULL); + window_calls = 0; + window_max_records = 0; + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &output, &output_size); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls == 2); + TEST_CHECK(window_max_records == WINDOW_TOTAL_RECORDS); + flb_free(output); + flb_processor_destroy(proc); + + flb_free(input); + + /* a buffer within one window keeps the single chunk path */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (i = 0; i < 300; i++) { + TEST_ASSERT(append_window_record(encoder, i) == 0); + } + proc = window_processor_create(config, &owner, "test_window_mark", NULL); + TEST_ASSERT(proc != NULL); + window_calls = 0; + window_max_records = 0; + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, + &output, &output_size); + TEST_CHECK(ret == 0); + TEST_CHECK(window_calls == 1); + TEST_CHECK(window_max_records == 300); + flb_free(output); + flb_processor_destroy(proc); + flb_log_event_encoder_destroy(encoder); + + cmt_destroy(owner.cmt); + window_config_destroy(config); +} + +/* + * The decoded chunk outlives the first unit's lock, so it must decode and + * encode with its own codec: another worker thread can reset the first + * unit's decoder while later units still read records from the chunk. + */ +static int codec_shared; + +static int probe_codec_and_stop(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_record *record; + + /* callbacks run under the unit lock, so the counter needs no atomics */ + if (chunk->log_decoder == ins->log_decoder || chunk->log_encoder == ins->log_encoder) { + codec_shared++; + } + + /* leave the remaining records for the next unit to decode */ + flb_mp_chunk_cobj_record_next(chunk, &record); + return FLB_PROCESSOR_SUCCESS; +} + +static struct flb_processor_plugin codec_probe_plugin = { + .name = "test_codec_probe", + .description = "checks the chunk codec and stops early", + .cb_process_logs = probe_codec_and_stop, + .flags = 0 +}; + +static struct flb_processor_plugin codec_window_probe_plugin = { + .name = "test_codec_window_probe", + .description = "checks the chunk codec and stops early, windowed", + .cb_process_logs = probe_codec_and_stop, + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS +}; + +struct codec_worker { + struct flb_processor *proc; + char *input; + size_t input_size; + int expected; + int failures; +}; + +static void *codec_worker_run(void *data) +{ + int i; + int ret; + int records; + void *output; + size_t output_size; + struct codec_worker *worker = data; + + for (i = 0; i < 50; i++) { + records = -1; + output = NULL; + ret = flb_processor_run_with_count(worker->proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + worker->input, worker->input_size, + &output, &output_size, -1, &records); + if (ret != 0 || records != worker->expected || output == NULL) { + worker->failures++; + } + if (output != NULL && output != worker->input) { + flb_free(output); + } + } + + return NULL; +} + +static void run_codec_workers(struct flb_config *config, char *first, char *second, + char *input, size_t input_size, int expected) +{ + int i; + struct flb_processor *proc; + struct flb_input_instance owner; + struct codec_worker workers[4]; + pthread_t threads[4]; + + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + proc = window_processor_create(config, &owner, first, second); + TEST_ASSERT(proc != NULL); + + for (i = 0; i < 4; i++) { + workers[i].proc = proc; + workers[i].input = input; + workers[i].input_size = input_size; + workers[i].expected = expected; + workers[i].failures = 0; + TEST_ASSERT(pthread_create(&threads[i], NULL, codec_worker_run, &workers[i]) == 0); + } + for (i = 0; i < 4; i++) { + pthread_join(threads[i], NULL); + TEST_CHECK(workers[i].failures == 0); + } + + flb_processor_destroy(proc); + cmt_destroy(owner.cmt); +} + +static void processor_chunk_codec_is_per_run(void) +{ + int i; + char *input; + size_t input_size; + struct flb_config *config; + struct flb_log_event_encoder *encoder; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + mk_list_add(&codec_probe_plugin._head, &config->processor_plugins); + mk_list_add(&codec_window_probe_plugin._head, &config->processor_plugins); + codec_shared = 0; + + /* single chunk path: the second unit decodes what the first one left */ + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (i = 0; i < 300; i++) { + TEST_ASSERT(append_window_record(encoder, i) == 0); + } + run_codec_workers(config, "test_codec_probe", "test_single_mark", + encoder->output_buffer, encoder->output_length, 300); + flb_log_event_encoder_destroy(encoder); + + /* windowed path */ + TEST_ASSERT(create_window_records(&input, &input_size) == 0); + run_codec_workers(config, "test_codec_window_probe", "test_window_mark", + input, input_size, WINDOW_TOTAL_RECORDS); + flb_free(input); + + TEST_CHECK(codec_shared == 0); + + mk_list_del(&codec_probe_plugin._head); + mk_list_del(&codec_window_probe_plugin._head); + window_config_destroy(config); +} + +/* removes group end markers, as a filtering processor can do */ +static int drop_group_ends(struct flb_processor_instance *ins, void *data, + const char *tag, int tag_len) +{ + int type; + struct cfl_list *head; + struct cfl_list *tmp; + struct flb_mp_chunk_cobj *chunk = data; + struct flb_mp_chunk_record *record; + + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + } + chunk->record_pos = NULL; + + cfl_list_foreach_safe(head, tmp, &chunk->records) { + record = cfl_list_entry(head, struct flb_mp_chunk_record, _head); + if (flb_log_event_decoder_get_record_type(&record->event, &type) == 0 && + type == FLB_LOG_EVENT_GROUP_END) { + flb_mp_chunk_cobj_record_destroy(chunk, record); + } + } + + return FLB_PROCESSOR_SUCCESS; +} + +static struct flb_processor_plugin drop_group_end_plugin = { + .name = "test_drop_group_end", + .description = "removes group end markers", + .cb_process_logs = drop_group_ends, + .flags = 0 +}; + +/* layout tokens: r record, S group start, E group end */ +static void run_group_layout(struct flb_config *config, char *unit, + const char *layout, const char *expected) +{ + int ret; + int type; + int length; + char token[16]; + char result[128]; + const char *cursor; + void *output = NULL; + size_t output_size = 0; + msgpack_object *seq; + struct flb_processor *proc; + struct flb_input_instance owner; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + struct flb_log_event_encoder *encoder; + + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + proc = window_processor_create(config, &owner, unit, NULL); + TEST_ASSERT(proc != NULL); + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + TEST_ASSERT(encoder != NULL); + for (cursor = layout; sscanf(cursor, "%15s%n", token, &length) == 1; cursor += length) { + if (token[0] == 'r') { + TEST_ASSERT(append_window_record(encoder, atoi(token + 1)) == 0); + } + else if (token[0] == 'S') { + TEST_ASSERT(flb_log_event_encoder_group_init(encoder) == 0); + TEST_ASSERT(flb_log_event_encoder_group_header_end(encoder) == 0); + } + else { + TEST_ASSERT(flb_log_event_encoder_group_end(encoder) == 0); + } + } + + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + encoder->output_buffer, encoder->output_length, + &output, &output_size); + TEST_CHECK(ret == 0); + + result[0] = '\0'; + if (output != NULL && output_size > 0) { + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == 0); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type == FLB_LOG_EVENT_GROUP_START) { + snprintf(token, sizeof(token), "S "); + } + else if (type == FLB_LOG_EVENT_GROUP_END) { + snprintf(token, sizeof(token), "E "); + } + else { + seq = &event.body->via.map.ptr[0].val; + snprintf(token, sizeof(token), "r%d ", (int) seq->via.i64); + } + strncat(result, token, sizeof(result) - strlen(result) - 1); + } + flb_log_event_decoder_destroy(&decoder); + } + if (strlen(result) > 0) { + result[strlen(result) - 1] = '\0'; + } + TEST_CHECK(strcmp(result, expected) == 0); + TEST_MSG("%s: '%s' produced '%s', expected '%s'", unit, layout, result, expected); + + if (output != NULL && output != encoder->output_buffer) { + flb_free(output); + } + flb_log_event_encoder_destroy(encoder); + flb_processor_destroy(proc); + cmt_destroy(owner.cmt); +} + +static void processor_group_boundaries_follow_the_input(void) +{ + struct flb_config *config; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + mk_list_add(&drop_group_end_plugin._head, &config->processor_plugins); + + /* a processor removing an end never leaves an orphan start behind */ + run_group_layout(config, "test_drop_group_end", "S r0 r1 E r2", "r0 r1 r2"); + /* groups split across appends keep their boundary markers */ + run_group_layout(config, "test_single_mark", "S r0 r1", "S r0 r1"); + run_group_layout(config, "test_single_mark", "r0 E S r1 E", "r0 E S r1 E"); + /* stray and empty markers of a balanced input are removed as before */ + run_group_layout(config, "test_single_mark", "S r0 E E r1", "S r0 E r1"); + + mk_list_del(&drop_group_end_plugin._head); + window_config_destroy(config); +} + TEST_LIST = { + {"processor_metrics_failure_clears_output", processor_metrics_failure_clears_output}, { "processor_private_inputs_use_main_loop", processor_private_inputs_use_main_loop }, { "processor", processor }, { "processor_grouped_filter_counters", processor_grouped_filter_counters }, { "processor_metrics_counters", processor_metrics_counters }, + { "processor_log_windows_match_single_chunk", processor_log_windows_match_single_chunk }, + { "processor_log_windows_drop_all", processor_log_windows_drop_all }, + { "processor_log_windows_need_every_unit", processor_log_windows_need_every_unit }, + { "processor_chunk_codec_is_per_run", processor_chunk_codec_is_per_run }, + { "processor_group_boundaries_follow_the_input", processor_group_boundaries_follow_the_input }, { 0 } };