From 1fe191a29dec7a15574fa6845f2536c769c76849 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 1 Oct 2026 22:05:16 -0600 Subject: [PATCH 01/13] processor: support fused raw log segments Signed-off-by: Eduardo Silva --- include/fluent-bit/flb_processor.h | 21 ++++ src/flb_processor.c | 151 ++++++++++++++++++++++++++++- 2 files changed, 167 insertions(+), 5 deletions(-) diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index 8b47488442a..6d0b6eab6ea 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -34,6 +34,21 @@ #define FLB_PROCESSOR_SUCCESS 0 #define FLB_PROCESSOR_FAILURE -1 +/* + * Optional raw callbacks execute a complete native segment in configuration + * order and preserve record cardinality. All units must use the same callback + * and enable logs_raw_enabled during init, with no conditions. The runner + * locks all participating units and accounts + * for each separately. Existing CFL callbacks remain the fallback for the whole + * segment. Raw callbacks never modify or release the input buffer. + * MODIFIED transfers a separately allocated output buffer to the caller. + * NOTOUCH, UNSUPPORTED and FAILURE leave output NULL/zero. UNSUPPORTED must + * discard any partial output and invoke no externally visible side effects. + */ +#define FLB_PROCESSOR_RAW_NOTOUCH 0 +#define FLB_PROCESSOR_RAW_MODIFIED 1 +#define FLB_PROCESSOR_RAW_UNSUPPORTED 2 + /* Processor event types */ #define FLB_PROCESSOR_LOGS 1 #define FLB_PROCESSOR_METRICS 2 @@ -175,6 +190,11 @@ struct flb_processor_plugin { const char *, int); + int (*cb_process_logs_raw) (struct flb_processor_instance **, size_t, + const void *, size_t, + void **, size_t *, + const char *, int); + int (*cb_process_metrics) (struct flb_processor_instance *, struct cmt *, /* in */ struct cmt **, /* out */ @@ -204,6 +224,7 @@ struct flb_processor_instance { int id; /* instance id */ int log_level; /* instance log level */ int event_type; /* event type */ + int logs_raw_enabled; /* set during init for compatible configurations */ char name[32]; /* numbered name */ char *alias; /* alias name */ void *context; /* Instance local context */ diff --git a/src/flb_processor.c b/src/flb_processor.c index 836fe42f66e..8e68a2353a1 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -1309,11 +1309,117 @@ int flb_processor_is_active(struct flb_processor *proc) #include /* - * This function will run all the processor units for the given tag and data, note - * that depending of the 'type', 'data' can reference a msgpack for logs, a CMetrics - * context for metrics, a 'CTraces' context for traces or a 'CProfiles' context for - * profiles. + * A native segment shares one representation. Only optimize it when every + * unit accepts the same raw callback; otherwise keep the existing CFL chain. */ +static int run_logs_raw_segment(struct flb_processor *proc, + struct flb_processor_unit *first, + const void *data, size_t bytes, int records, + const char *tag, size_t tag_len, + void **out_buf, size_t *out_size, + struct mk_list **last) +{ + int result; + int out_records; + int error; + size_t count; + size_t index; + size_t locked; + struct mk_list *head; + struct flb_processor_unit *unit; + struct flb_processor_instance *initial; + struct flb_processor_instance *single; + struct flb_processor_instance **instances; + const char *scope; + const char *owner; + + initial = first->ctx; + if (first->condition != NULL || !initial->logs_raw_enabled || + initial->p->cb_process_logs_raw == NULL) { + return FLB_PROCESSOR_RAW_UNSUPPORTED; + } + count = 0; + for (head = &first->_head; head != &proc->logs; head = head->next) { + unit = mk_list_entry(head, struct flb_processor_unit, _head); + if (unit->unit_type == FLB_PROCESSOR_UNIT_FILTER) { + break; + } + if (unit->condition != NULL || + !((struct flb_processor_instance *) unit->ctx)->logs_raw_enabled || + ((struct flb_processor_instance *) unit->ctx)->p->cb_process_logs_raw != + initial->p->cb_process_logs_raw) { + return FLB_PROCESSOR_RAW_UNSUPPORTED; + } + count++; + } + single = initial; + instances = &single; + if (count > 1) { + instances = flb_calloc(count, sizeof(struct flb_processor_instance *)); + if (instances == NULL) { + flb_errno(); + return FLB_PROCESSOR_RAW_UNSUPPORTED; + } + } + index = 0; + locked = 0; + result = FLB_PROCESSOR_FAILURE; + for (head = &first->_head; index < count; head = head->next) { + unit = mk_list_entry(head, struct flb_processor_unit, _head); + instances[index] = unit->ctx; + if (acquire_lock(&unit->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY) != FLB_TRUE) { + goto cleanup; + } + locked++; + index++; + *last = head; + } + result = initial->p->cb_process_logs_raw(instances, count, data, bytes, + out_buf, out_size, tag, tag_len); + if (result == FLB_PROCESSOR_RAW_MODIFIED) { + /* A fused segment cannot attribute additions/drops to individual units. */ + if (*out_buf == NULL || *out_buf == data || + flb_mp_count_log_records(*out_buf, *out_size) != records) { + if (*out_buf != data) { + flb_free(*out_buf); + } + *out_buf = NULL; + *out_size = 0; + result = FLB_PROCESSOR_FAILURE; + } + } + else if (*out_buf != NULL || *out_size != 0 || + (result != FLB_PROCESSOR_RAW_NOTOUCH && + result != FLB_PROCESSOR_RAW_UNSUPPORTED)) { + if (*out_buf != data) { + flb_free(*out_buf); + } + *out_buf = NULL; + *out_size = 0; + result = FLB_PROCESSOR_FAILURE; + } + +cleanup: + scope = processor_metrics_scope(proc); + owner = processor_metrics_owner(proc); + error = (result == FLB_PROCESSOR_FAILURE); + out_records = records; + for (index = 0; index < locked; index++) { + unit = instances[index]->pu; + if (result != FLB_PROCESSOR_RAW_UNSUPPORTED) { + processor_metrics_update(proc, unit, scope, owner, "logs", + records, out_records, FLB_TRUE, error); + } + release_lock(&unit->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + } + if (count > 1) { + flb_free(instances); + } + return result; +} + /* * Decoder and encoder owned by one processor run. The decoded chunk lives * across several units, so it must not borrow the first unit's decoder: its @@ -1666,6 +1772,10 @@ int flb_processor_run(struct flb_processor *proc, data, data_size, out_buf, out_size, -1, NULL); } +/* + * Run processor units for a tag and signal. Data contains MessagePack logs, + * or a CMetrics, CTraces or CProfiles context for the other signal types. + */ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, size_t starting_stage, int type, @@ -1680,6 +1790,7 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, int unit_out_items; int item_metrics_available; int finalize; + int raw_result; int log_buffer_normalized = FLB_FALSE; int log_buffer_records = input_records; void *cur_buf = NULL; @@ -1693,6 +1804,7 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, struct flb_filter_instance *f_ins; struct flb_processor_instance *p_ins; struct flb_mp_chunk_cobj *chunk_cobj = NULL; + struct mk_list *raw_last; struct mk_list *run_end; int run_units; int window_records; @@ -1759,7 +1871,11 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, item_metrics_available = FLB_FALSE; if (type == FLB_PROCESSOR_LOGS) { - if (!chunk_cobj && log_buffer_records >= 0) { + if (chunk_cobj != NULL && + chunk_cobj->log_decoder->offset == chunk_cobj->log_decoder->length) { + unit_in_items = flb_mp_chunk_cobj_count_log_records(chunk_cobj); + } + else if (!chunk_cobj && log_buffer_records >= 0) { unit_in_items = log_buffer_records; } else { @@ -1769,6 +1885,31 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, item_metrics_available = FLB_TRUE; } + if (type == FLB_PROCESSOR_LOGS && chunk_cobj == NULL && + pu->unit_type == FLB_PROCESSOR_UNIT_NATIVE) { + raw_last = head; + raw_result = run_logs_raw_segment(proc, pu, cur_buf, cur_size, unit_in_items, + tag, tag_len, &tmp_buf, &tmp_size, &raw_last); + if (raw_result == FLB_PROCESSOR_FAILURE) { + if (cur_buf != data) { + flb_free(cur_buf); + } + return -1; + } + if (raw_result != FLB_PROCESSOR_RAW_UNSUPPORTED) { + if (raw_result == FLB_PROCESSOR_RAW_MODIFIED) { + if (cur_buf != data) { + flb_free(cur_buf); + } + cur_buf = tmp_buf; + cur_size = tmp_size; + } + log_buffer_records = unit_in_items; + head = raw_last; + continue; + } + } + /* large buffers run record local native units in bounded windows */ if (type == FLB_PROCESSOR_LOGS && chunk_cobj == NULL && pu->unit_type == FLB_PROCESSOR_UNIT_NATIVE && From 9f4d5fa162755d7d3fca5374c0c8c528e3bb3750 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 1 Oct 2026 22:05:16 -0600 Subject: [PATCH 02/13] processor_content_modifier: fuse compatible body edits Signed-off-by: Eduardo Silva --- .../processor_content_modifier/CMakeLists.txt | 1 + plugins/processor_content_modifier/cm.c | 2 + plugins/processor_content_modifier/cm.h | 7 + .../processor_content_modifier/cm_msgpack.c | 231 ++++++++++++++++++ 4 files changed, 241 insertions(+) create mode 100644 plugins/processor_content_modifier/cm_msgpack.c diff --git a/plugins/processor_content_modifier/CMakeLists.txt b/plugins/processor_content_modifier/CMakeLists.txt index 681d2cd58df..6ccbad6e202 100644 --- a/plugins/processor_content_modifier/CMakeLists.txt +++ b/plugins/processor_content_modifier/CMakeLists.txt @@ -1,6 +1,7 @@ set(src cm_config.c cm_logs.c + cm_msgpack.c cm_metrics.c cm_traces.c cm_opentelemetry.c diff --git a/plugins/processor_content_modifier/cm.c b/plugins/processor_content_modifier/cm.c index ebc34831b2e..5482256b9ef 100644 --- a/plugins/processor_content_modifier/cm.c +++ b/plugins/processor_content_modifier/cm.c @@ -41,6 +41,7 @@ static int cb_init(struct flb_processor_instance *ins, void *source_plugin_insta } flb_processor_instance_set_context(ins, ctx); + ins->logs_raw_enabled = cm_logs_raw_supported(ins); return FLB_PROCESSOR_SUCCESS; } @@ -163,6 +164,7 @@ struct flb_processor_plugin processor_content_modifier_plugin = { .description = "Modify the content of Logs, Metrics and Traces", .cb_init = cb_init, .cb_process_logs = cb_process_logs, + .cb_process_logs_raw = cm_logs_process_raw, .cb_process_metrics = cb_process_metrics, .cb_process_traces = cb_process_traces, .cb_exit = cb_exit, diff --git a/plugins/processor_content_modifier/cm.h b/plugins/processor_content_modifier/cm.h index 65e7664d755..90cd3446f56 100644 --- a/plugins/processor_content_modifier/cm.h +++ b/plugins/processor_content_modifier/cm.h @@ -107,6 +107,13 @@ struct content_modifier_ctx { }; /* Export telemetry functions */ +int cm_logs_raw_supported(struct flb_processor_instance *ins); + +int cm_logs_process_raw(struct flb_processor_instance **instances, size_t instance_count, + const void *data, size_t bytes, + void **out_buf, size_t *out_size, + const char *tag, int tag_len); + int cm_logs_process(struct flb_processor_instance *ins, struct content_modifier_ctx *ctx, struct flb_mp_chunk_cobj *chunk_cobj, diff --git a/plugins/processor_content_modifier/cm_msgpack.c b/plugins/processor_content_modifier/cm_msgpack.c new file mode 100644 index 00000000000..2c9a014c7d3 --- /dev/null +++ b/plugins/processor_content_modifier/cm_msgpack.c @@ -0,0 +1,231 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +#include "cm.h" +#include +#include +#include +#include + +/* Fall back for shapes that the existing CFL conversion cannot represent. */ +static int object_supported(msgpack_object *object, size_t depth) +{ + size_t index; + + if (depth > 32 || object->type == MSGPACK_OBJECT_EXT) { + return FLB_FALSE; + } + if (object->type == MSGPACK_OBJECT_MAP) { + for (index = 0; index < object->via.map.size; index++) { + if (object->via.map.ptr[index].key.type != MSGPACK_OBJECT_STR || + !object_supported(&object->via.map.ptr[index].val, depth + 1)) { + return FLB_FALSE; + } + } + } + else if (object->type == MSGPACK_OBJECT_ARRAY) { + for (index = 0; index < object->via.array.size; index++) { + if (!object_supported(&object->via.array.ptr[index], depth + 1)) { + return FLB_FALSE; + } + } + } + return FLB_TRUE; +} + +static void string_object(msgpack_object *object, cfl_sds_t value) +{ + object->type = MSGPACK_OBJECT_STR; + object->via.str.ptr = value; + object->via.str.size = cfl_sds_len(value); +} + +int cm_logs_raw_supported(struct flb_processor_instance *ins) +{ + struct content_modifier_ctx *ctx; + + ctx = ins->context; + return ctx != NULL && ctx->context_type == CM_CONTEXT_LOG_BODY && + (ctx->action_type == CM_ACTION_INSERT || ctx->action_type == CM_ACTION_UPSERT || + ctx->action_type == CM_ACTION_DELETE || ctx->action_type == CM_ACTION_RENAME); +} + +int cm_logs_process_raw(struct flb_processor_instance **instances, size_t instance_count, + const void *data, size_t bytes, + void **out_buf, size_t *out_size, + const char *tag, int tag_len) +{ + int ret; + int result; + int changed; + int record_changed; + int record_type; + size_t index; + size_t consumed; + size_t action_index; + size_t match; + size_t capacity; + size_t required; + size_t count; + size_t key_length; + struct content_modifier_ctx *ctx; + struct flb_log_event_decoder decoder; + struct flb_log_event event; + msgpack_sbuffer output; + msgpack_packer packer; + msgpack_object root; + msgpack_object elements[2]; + msgpack_object_kv *pairs; + msgpack_object_kv *resized; + + (void) tag; + (void) tag_len; + if (instance_count == 0 || instance_count > UINT32_MAX) { + return FLB_PROCESSOR_RAW_UNSUPPORTED; + } + for (action_index = 0; action_index < instance_count; action_index++) { + if (!cm_logs_raw_supported(instances[action_index])) { + return FLB_PROCESSOR_RAW_UNSUPPORTED; + } + } + + ret = flb_log_event_decoder_init(&decoder, (char *) data, bytes); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + return FLB_PROCESSOR_RAW_UNSUPPORTED; + } + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + msgpack_sbuffer_init(&output); + msgpack_packer_init(&packer, &output, msgpack_sbuffer_write); + pairs = NULL; + capacity = 0; + changed = FLB_FALSE; + consumed = 0; + result = FLB_PROCESSOR_RAW_UNSUPPORTED; + + while ((ret = flb_log_event_decoder_next(&decoder, &event)) == FLB_EVENT_DECODER_SUCCESS) { + /* The decoder can skip invalid markers. Let the existing path handle them. */ + if (decoder.record_base != (const char *) data + consumed) { + goto cleanup; + } + consumed += decoder.record_length; + ret = flb_log_event_decoder_get_record_type(&event, &record_type); + if (ret != FLB_EVENT_DECODER_SUCCESS || record_type != FLB_LOG_EVENT_NORMAL || + event.format != FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2 || + event.body->type != MSGPACK_OBJECT_MAP || + !object_supported(event.body, 0) || !object_supported(event.metadata, 0)) { + goto cleanup; + } + + /* An action can add at most one field. Reuse the largest scratch map. */ + if (instance_count > UINT32_MAX - event.body->via.map.size) { + result = FLB_PROCESSOR_FAILURE; + goto cleanup; + } + required = (size_t) event.body->via.map.size + instance_count; + if (required > SIZE_MAX / sizeof(msgpack_object_kv)) { + result = FLB_PROCESSOR_FAILURE; + goto cleanup; + } + if (required > capacity) { + resized = flb_realloc(pairs, required * sizeof(msgpack_object_kv)); + if (resized == NULL) { + result = FLB_PROCESSOR_FAILURE; + goto cleanup; + } + pairs = resized; + capacity = required; + } + count = event.body->via.map.size; + if (count > 0) { + memcpy(pairs, event.body->via.map.ptr, count * sizeof(msgpack_object_kv)); + } + record_changed = FLB_FALSE; + for (action_index = 0; action_index < instance_count; action_index++) { + ctx = instances[action_index]->context; + key_length = cfl_sds_len(ctx->key); + match = SIZE_MAX; + for (index = 0; index < count; index++) { + if (pairs[index].key.via.str.size == key_length && + strncmp(pairs[index].key.via.str.ptr, ctx->key, key_length) == 0) { + match = index; + break; + } + } + if ((ctx->action_type == CM_ACTION_INSERT && match != SIZE_MAX) || + ((ctx->action_type == CM_ACTION_DELETE || ctx->action_type == CM_ACTION_RENAME) && + match == SIZE_MAX)) { + continue; + } + if (ctx->action_type == CM_ACTION_RENAME) { + string_object(&pairs[match].key, ctx->value); + } + else { + if (match != SIZE_MAX) { + memmove(&pairs[match], &pairs[match + 1], + (count - match - 1) * sizeof(msgpack_object_kv)); + count--; + } + if (ctx->action_type == CM_ACTION_INSERT || ctx->action_type == CM_ACTION_UPSERT) { + string_object(&pairs[count].key, ctx->key); + string_object(&pairs[count].val, ctx->value); + count++; + } + } + record_changed = FLB_TRUE; + } + if (!record_changed) { + if (msgpack_sbuffer_write(&output, decoder.record_base, decoder.record_length) != 0) { + result = FLB_PROCESSOR_FAILURE; + goto cleanup; + } + continue; + } + + root = *event.root; + memcpy(elements, root.via.array.ptr, sizeof(elements)); + elements[1] = *event.body; + elements[1].via.map.ptr = pairs; + elements[1].via.map.size = count; + root.via.array.ptr = elements; + if (msgpack_pack_object(&packer, root) != 0) { + result = FLB_PROCESSOR_FAILURE; + goto cleanup; + } + changed = FLB_TRUE; + } + + if (consumed != bytes || + flb_log_event_decoder_get_last_result(&decoder) != FLB_EVENT_DECODER_SUCCESS) { + goto cleanup; + } + result = FLB_PROCESSOR_RAW_NOTOUCH; + if (changed) { + *out_buf = output.data; + *out_size = output.size; + output.data = NULL; + result = FLB_PROCESSOR_RAW_MODIFIED; + } + +cleanup: + msgpack_sbuffer_destroy(&output); + flb_free(pairs); + flb_log_event_decoder_destroy(&decoder); + return result; +} From 35e3561578dae1300b8d347657443e05e08ce7d1 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 1 Oct 2026 22:05:16 -0600 Subject: [PATCH 03/13] tests: internal: cover native chain drop accounting Signed-off-by: Eduardo Silva --- tests/internal/processor.c | 113 +++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/tests/internal/processor.c b/tests/internal/processor.c index d77fba9d86f..91824e7f3e3 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -1441,7 +1441,120 @@ static void processor_group_boundaries_follow_the_input(void) window_config_destroy(config); } +static int native_drop_second_record(struct flb_processor_instance *ins, + void *data, const char *tag, int tag_len) +{ + int ret; + int type; + int count; + struct flb_mp_chunk_cobj *chunk; + struct flb_mp_chunk_record *record; + + chunk = data; + count = 0; + while ((ret = flb_mp_chunk_cobj_record_next(chunk, &record)) == FLB_MP_CHUNK_RECORD_OK) { + TEST_CHECK(flb_log_event_decoder_get_record_type(&record->event, &type) == 0); + if (type == FLB_LOG_EVENT_NORMAL && ++count == 2) { + TEST_CHECK(flb_mp_chunk_cobj_record_destroy(chunk, record) == 0); + } + } + return ret == FLB_MP_CHUNK_RECORD_EOF ? FLB_PROCESSOR_SUCCESS : FLB_PROCESSOR_FAILURE; +} + +static void processor_native_chain_counters(void) +{ + int grouped; + int root_type; + char *data; + char *resized; + size_t bytes; + size_t out_size; + void *out; + double value; + struct flb_config *config; + struct flb_processor *proc; + struct flb_processor_unit *first; + struct flb_processor_unit *second; + struct flb_processor_instance *ins; + struct flb_processor_plugin *original; + struct flb_processor_plugin drop; + struct flb_input_instance owner; + const char *json = "[[1700000000,{}],{\"message\":\"first\"}]" + "[[1700000001,{}],{\"message\":\"second\"}]"; + + flb_init_env(); + for (grouped = 0; grouped <= 1; grouped++) { + config = flb_config_init(); + TEST_CHECK(config != NULL); + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "unit_input.0"); + owner.cmt = cmt_create(); + TEST_CHECK(owner.cmt != NULL); + proc = flb_processor_create(config, "native_counters", &owner, FLB_PLUGIN_INPUT); + TEST_CHECK(proc != NULL); + first = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "content_modifier"); + second = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "content_modifier"); + TEST_CHECK(first != NULL && second != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(first, "action", "insert") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(first, "key", "unused") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(first, "value", "unused") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(second, "action", "insert") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(second, "key", "after_drop") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(second, "value", "yes") == 0); + TEST_CHECK(flb_processor_init(proc) == 0); + ins = first->ctx; + original = ins->p; + drop = *original; + drop.cb_process_logs_raw = NULL; + drop.cb_process_logs = native_drop_second_record; + ins->p = &drop; + if (grouped) { + TEST_CHECK(create_grouped_msgpack_records(&data, &bytes) == 0); + resized = flb_realloc(data, bytes * 2); + TEST_CHECK(resized != NULL); + data = resized; + memcpy(data + bytes, data, bytes); + bytes *= 2; + } + else { + TEST_CHECK(flb_pack_json(json, strlen(json), &data, &bytes, &root_type, NULL) == 0); + } + TEST_CHECK(flb_mp_count_log_records(data, bytes) == 2); + out = NULL; + TEST_CHECK(flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + data, bytes, &out, &out_size) == 0); + TEST_CHECK(flb_mp_count_log_records(out, out_size) == 1); + TEST_CHECK(get_counter_value_5(proc->cmt_items_in, "input", "unit_input.0", + "content_modifier", "0", "logs", &value) == 0); + TEST_CHECK(value == 2.0); + TEST_CHECK(get_counter_value_5(proc->cmt_items_out, "input", "unit_input.0", + "content_modifier", "0", "logs", &value) == 0); + TEST_CHECK(value == 1.0); + TEST_CHECK(get_counter_value_5(proc->cmt_items_in, "input", "unit_input.0", + "content_modifier", "1", "logs", &value) == 0); + TEST_CHECK(value == 1.0); + TEST_CHECK(get_counter_value_5(proc->cmt_items_out, "input", "unit_input.0", + "content_modifier", "1", "logs", &value) == 0); + TEST_CHECK(value == 1.0); + TEST_CHECK(get_counter_value_5(proc->cmt_items_drop, "input", "unit_input.0", + "content_modifier", "0", "logs", &value) == 0); + TEST_CHECK(value == 1.0); + TEST_CHECK(get_counter_value_5_or_zero(proc->cmt_items_drop, "input", "unit_input.0", + "content_modifier", "1", "logs", &value) == 0); + TEST_CHECK(value == 0.0); + ins->p = original; + if (out != data) { + flb_free(out); + } + flb_free(data); + flb_processor_destroy(proc); + cmt_destroy(owner.cmt); + flb_config_exit(config); + } +} + TEST_LIST = { + { "processor_native_chain_counters", processor_native_chain_counters }, {"processor_metrics_failure_clears_output", processor_metrics_failure_clears_output}, { "processor_private_inputs_use_main_loop", processor_private_inputs_use_main_loop }, { "processor", processor }, From c62019a3be838fbafeb817ba0dfdd6951ee3e32c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 1 Oct 2026 22:05:16 -0600 Subject: [PATCH 04/13] tests: runtime: cover raw processor equivalence and ownership Signed-off-by: Eduardo Silva --- tests/runtime/processor_content_modifier.c | 305 +++++++++++++++++++++ 1 file changed, 305 insertions(+) diff --git a/tests/runtime/processor_content_modifier.c b/tests/runtime/processor_content_modifier.c index fd494c5e6fd..5ecc1c6e8c7 100644 --- a/tests/runtime/processor_content_modifier.c +++ b/tests/runtime/processor_content_modifier.c @@ -1876,7 +1876,312 @@ static void flb_logs_otel_log_attributes_invalid_otlp_metadata() processor_test_destroy(ctx); } +/* Compare the optimized path with the existing CFL path on identical batches. */ +static void check_raw_chain_equivalence(const char *action, const char *json, + int expected_result, size_t modifiers) +{ + int ret; + int root_type; + int raw_result; + int expected_ret; + char *data; + char *saved; + char *actual_json; + char *expected_json; + size_t bytes; + size_t actual_size; + size_t expected_size; + size_t probe_size; + size_t index; + struct flb_processor_instance *instances[6]; + void *actual; + void *expected; + void *probe; + flb_ctx_t *flb; + struct flb_processor *proc; + struct flb_processor_unit *unit; + struct flb_processor_instance *ins; + struct flb_processor_plugin *original; + struct flb_processor_plugin fallback; + struct flb_log_event_decoder actual_decoder; + struct flb_log_event_decoder expected_decoder; + struct flb_log_event actual_event; + struct flb_log_event expected_event; + + flb = flb_create(); + TEST_CHECK(flb != NULL); + proc = flb_processor_create(flb->config, "raw_equivalence", NULL, 0); + TEST_CHECK(proc != NULL); + TEST_CHECK(modifiers > 0 && modifiers <= 6); + for (index = 0; index < modifiers; index++) { + unit = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "content_modifier"); + TEST_CHECK(unit != NULL); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "action", action) == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "context", "body") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "key", "target") == 0); + if (strcmp(action, "delete") != 0) { + TEST_CHECK(flb_processor_unit_set_property_str(unit, "value", "replacement") == 0); + } + instances[index] = unit->ctx; + } + TEST_CHECK(flb_processor_init(proc) == 0); + ins = unit->ctx; + original = ins->p; + fallback = *original; + fallback.cb_process_logs_raw = NULL; + + ret = flb_pack_json(json, strlen(json), &data, &bytes, &root_type, NULL); + TEST_CHECK(ret == 0); + saved = flb_malloc(bytes); + TEST_CHECK(saved != NULL); + memcpy(saved, data, bytes); + probe = NULL; + probe_size = 0; + raw_result = original->cb_process_logs_raw(instances, modifiers, data, bytes, + &probe, &probe_size, "test", 4); + TEST_CHECK(raw_result == expected_result); + TEST_CHECK(memcmp(data, saved, bytes) == 0); + flb_free(probe); + + actual = NULL; + expected = NULL; + TEST_CHECK(flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + data, bytes, &actual, &actual_size) == 0); + ins->p = &fallback; + TEST_CHECK(flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + data, bytes, &expected, &expected_size) == 0); + ins->p = original; + TEST_CHECK(memcmp(data, saved, bytes) == 0); + + TEST_CHECK(flb_log_event_decoder_init(&actual_decoder, actual, actual_size) == 0); + TEST_CHECK(flb_log_event_decoder_init(&expected_decoder, expected, expected_size) == 0); + flb_log_event_decoder_read_groups(&actual_decoder, FLB_TRUE); + flb_log_event_decoder_read_groups(&expected_decoder, FLB_TRUE); + while ((ret = flb_log_event_decoder_next(&actual_decoder, &actual_event)) == 0) { + expected_ret = flb_log_event_decoder_next(&expected_decoder, &expected_event); + TEST_CHECK(expected_ret == 0); + if (expected_ret != 0) { + break; + } + TEST_CHECK(actual_event.timestamp.tm.tv_sec == expected_event.timestamp.tm.tv_sec); + TEST_CHECK(actual_event.timestamp.tm.tv_nsec == expected_event.timestamp.tm.tv_nsec); + actual_json = flb_msgpack_to_json_str(4096, actual_event.body, FLB_TRUE); + expected_json = flb_msgpack_to_json_str(4096, expected_event.body, FLB_TRUE); + TEST_CHECK(actual_json != NULL && expected_json != NULL); + TEST_CHECK(strcmp(actual_json, expected_json) == 0); + flb_free(actual_json); + flb_free(expected_json); + actual_json = flb_msgpack_to_json_str(4096, actual_event.metadata, FLB_TRUE); + expected_json = flb_msgpack_to_json_str(4096, expected_event.metadata, FLB_TRUE); + TEST_CHECK(actual_json != NULL && expected_json != NULL); + TEST_CHECK(strcmp(actual_json, expected_json) == 0); + flb_free(actual_json); + flb_free(expected_json); + } + TEST_CHECK(flb_log_event_decoder_get_last_result(&actual_decoder) == 0); + TEST_CHECK(flb_log_event_decoder_next(&expected_decoder, &expected_event) != 0); + TEST_CHECK(flb_log_event_decoder_get_last_result(&expected_decoder) == 0); + flb_log_event_decoder_destroy(&actual_decoder); + flb_log_event_decoder_destroy(&expected_decoder); + if (actual != data) { + flb_free(actual); + } + if (expected != data) { + flb_free(expected); + } + flb_free(saved); + flb_free(data); + flb_processor_destroy(proc); + flb_destroy(flb); +} + +static void check_raw_equivalence(const char *action, const char *json, int expected_result) +{ + check_raw_chain_equivalence(action, json, expected_result, 1); +} + +static void flb_logs_raw_chain_equivalence(void) +{ + const char *actions[] = {"insert", "upsert", "delete", "rename"}; + const char *json = + "[[1700000000,{\"origin\":\"test\"}]," + "{\"target\":\"first\",\"target\":\"second\",\"replacement\":\"collision\"}]" + "[[1700000001,{}],{\"nested\":{\"values\":[true,null,1,1.5]}}]" + "[[1700000002,{}],{}]"; + size_t index; + size_t count; + + for (index = 0; index < sizeof(actions) / sizeof(actions[0]); index++) { + for (count = 3; count <= 6; count += 3) { + check_raw_chain_equivalence(actions[index], json, FLB_PROCESSOR_RAW_MODIFIED, count); + check_raw_chain_equivalence(actions[index], + "[[1700000000,{}],{\"target\":\"plain\"}]" + "[[-1,{}],{}]" + "[[1700000001,{}],{\"target\":\"grouped\"}]" + "[[-2,{}],{}]", FLB_PROCESSOR_RAW_UNSUPPORTED, count); + } + } +} + +static void flb_logs_raw_equivalence(void) +{ + const char *actions[] = {"insert", "upsert", "delete", "rename"}; + const char *batch = + "[[1700000000.125,{\"source\":\"test\"}]," + "{\"target\":\"first\",\"target\":\"second\",\"replacement\":\"existing\",\"nested\":{\"x\":[null,true,1,1.5]}}]" + "[[1700000001,{}],{\"other\":\"unchanged\"}]" + "[[1700000002,{}],{}]"; + size_t index; + + for (index = 0; index < sizeof(actions) / sizeof(actions[0]); index++) { + check_raw_equivalence(actions[index], batch, FLB_PROCESSOR_RAW_MODIFIED); + check_raw_equivalence(actions[index], "[1700000000,{\"target\":\"legacy\"}]", + FLB_PROCESSOR_RAW_UNSUPPORTED); + check_raw_equivalence(actions[index], + "[[-1,{\"resource\":\"test\"}],{\"scope\":\"group\"}]" + "[[1700000000,{}],{\"target\":\"grouped\"}]" + "[[-2,{}],{}]", FLB_PROCESSOR_RAW_UNSUPPORTED); + /* Unsupported records after a modified prefix must discard the private output. */ + check_raw_equivalence(actions[index], + "[[1700000000,{}],{\"target\":\"plain\"}]" + "[[-1,{}],{}]" + "[[1700000001,{}],{\"target\":\"grouped\"}]" + "[[-2,{}],{}]", FLB_PROCESSOR_RAW_UNSUPPORTED); + check_raw_equivalence(actions[index], + "[[1700000000,{}],{\"target\":\"valid\"}]" + "[[-3,{}],{\"target\":\"invalid marker\"}]", + FLB_PROCESSOR_RAW_UNSUPPORTED); + } + check_raw_equivalence("insert", "[[1700000000,{}],{\"target\":\"keep\"}]", + FLB_PROCESSOR_RAW_NOTOUCH); + check_raw_equivalence("delete", "[[1700000000,{}],{\"other\":\"keep\"}]", + FLB_PROCESSOR_RAW_NOTOUCH); + check_raw_equivalence("rename", "[[1700000000,{}],{}]", FLB_PROCESSOR_RAW_NOTOUCH); +} + +static void flb_logs_raw_invalid_batch(void) +{ + int ret; + size_t bytes; + size_t out_size; + int root_type; + char *data; + void *out; + flb_ctx_t *flb; + struct flb_processor *proc; + struct flb_processor_unit *unit; + struct flb_processor_instance *ins; + const char *json = "[[1700000000,{}],{\"target\":\"valid\"}]" + "[[1700000001,{}],{\"target\":\"truncated\"}]"; + + flb = flb_create(); + proc = flb_processor_create(flb->config, "raw_invalid", NULL, 0); + unit = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "content_modifier"); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "action", "upsert") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "key", "target") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "value", "replacement") == 0); + TEST_CHECK(flb_processor_init(proc) == 0); + ins = unit->ctx; + TEST_CHECK(flb_pack_json(json, strlen(json), &data, &bytes, &root_type, NULL) == 0); + out = NULL; + out_size = 0; + ret = ins->p->cb_process_logs_raw(&ins, 1, data, bytes - 1, &out, &out_size, "test", 4); + TEST_CHECK(ret == FLB_PROCESSOR_RAW_UNSUPPORTED); + TEST_CHECK(out == NULL && out_size == 0); + flb_free(data); + flb_processor_destroy(proc); + flb_destroy(flb); +} + +static int raw_test_result; + +static int raw_test_callback(struct flb_processor_instance **instances, size_t count, + const void *data, size_t bytes, + void **out_buf, size_t *out_size, + const char *tag, int tag_len) +{ + *out_buf = NULL; + *out_size = 0; + return raw_test_result; +} + +static void flb_logs_raw_results(void) +{ + int ret; + int root_type; + size_t bytes; + size_t out_size; + size_t index; + char *data; + char *json; + void *out; + flb_ctx_t *flb; + struct flb_processor *proc; + struct flb_processor_unit *unit; + struct flb_processor_instance *ins; + struct flb_processor_plugin *original; + struct flb_processor_plugin mock; + int results[] = {FLB_PROCESSOR_RAW_NOTOUCH, FLB_PROCESSOR_RAW_UNSUPPORTED, + FLB_PROCESSOR_RAW_MODIFIED, FLB_PROCESSOR_FAILURE}; + const char *input = "[[1700000000,{}],{\"message\":\"test\"}]"; + + for (index = 0; index < sizeof(results) / sizeof(results[0]); index++) { + flb = flb_create(); + proc = flb_processor_create(flb->config, "raw_results", NULL, 0); + unit = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "modify"); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "add", "first yes") == 0); + unit = flb_processor_unit_create(proc, FLB_PROCESSOR_LOGS, "content_modifier"); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "action", "insert") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "key", "second") == 0); + TEST_CHECK(flb_processor_unit_set_property_str(unit, "value", "yes") == 0); + TEST_CHECK(flb_processor_init(proc) == 0); + ins = unit->ctx; + original = ins->p; + mock = *original; + mock.cb_process_logs_raw = raw_test_callback; + ins->p = &mock; + raw_test_result = results[index]; + TEST_CHECK(flb_pack_json(input, strlen(input), &data, &bytes, &root_type, NULL) == 0); + out = NULL; + out_size = 0; + ret = flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + data, bytes, &out, &out_size); + if (raw_test_result == FLB_PROCESSOR_FAILURE) { + TEST_CHECK(ret == -1); + TEST_CHECK(out == NULL); + } + else if (raw_test_result == FLB_PROCESSOR_RAW_MODIFIED) { + TEST_CHECK(ret == -1); + TEST_CHECK(out == NULL && out_size == 0); + } + else { + TEST_CHECK(ret == 0); + json = flb_msgpack_raw_to_json_sds(out, out_size, FLB_TRUE); + TEST_CHECK(json != NULL); + TEST_CHECK(strstr(json, "first") != NULL); + if (raw_test_result == FLB_PROCESSOR_RAW_UNSUPPORTED) { + TEST_CHECK(strstr(json, "second") != NULL); + } + else { + TEST_CHECK(strstr(json, "second") == NULL); + } + flb_sds_destroy(json); + } + ins->p = original; + if (out != data) { + flb_free(out); + } + flb_free(data); + flb_processor_destroy(proc); + flb_destroy(flb); + } +} + TEST_LIST = { + {"logs.raw.chain_equivalence", flb_logs_raw_chain_equivalence}, + {"logs.raw.results", flb_logs_raw_results}, + {"logs.raw.equivalence", flb_logs_raw_equivalence}, + {"logs.raw.invalid_batch", flb_logs_raw_invalid_batch}, {"logs.action.insert" , flb_logs_action_insert }, {"logs.action.delete" , flb_logs_action_delete }, {"logs.action.rename" , flb_logs_action_rename }, From 0c245b38c34f16e5d895f17b72802f443e31dde4 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 1 Oct 2026 22:05:16 -0600 Subject: [PATCH 05/13] tests: integration: cover fused and fallback processor chains Signed-off-by: Eduardo Silva --- .../tests/test_content_modifier.py | 327 ++++++++++++++++++ .../tests/test_grouped_logs.py | 175 ++++++++++ 2 files changed, 502 insertions(+) create mode 100644 tests/integration/scenarios/processor_content_modifier/tests/test_content_modifier.py create mode 100644 tests/integration/scenarios/processor_content_modifier/tests/test_grouped_logs.py diff --git a/tests/integration/scenarios/processor_content_modifier/tests/test_content_modifier.py b/tests/integration/scenarios/processor_content_modifier/tests/test_content_modifier.py new file mode 100644 index 00000000000..f7bfb8fba05 --- /dev/null +++ b/tests/integration/scenarios/processor_content_modifier/tests/test_content_modifier.py @@ -0,0 +1,327 @@ +"""Content modifier correctness across input/output processing and fan-out.""" + +from copy import deepcopy +from concurrent.futures import ThreadPoolExecutor +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import hashlib +import re +import threading + +import pytest +import requests +import yaml + +from utils.test_service import FluentBitTestService + + +RECORDS = [ + {"id": 1, "target": "original", "nested": {"array": [None, True, 1, 1.5]}}, + {"id": 2, "other": "untouched"}, +] + + +def processor_metrics(text, scope, owner="http.0"): + values = {} + for line in text.splitlines(): + if not line.startswith("fluentbit_processor_") or "{" not in line: + continue + name, body = line.split("{", 1) + labels, value = body.split("}", 1) + labels = dict(re.findall(r'(\w+)="([^"]*)"', labels)) + if labels.get("scope") == scope and labels.get("owner") == owner: + values[(name, int(labels["stage"]))] = float(value.split()[0]) + return values + + +def run_pipeline(tmp_path, processors, expected, *, location="input", storage="memory", + records=None, copies=1, stage_counts=None): + records = RECORDS if records is None else records + total_records = len(records) * copies + stage_counts = ([(len(records), len(records))] * len(processors) + if stage_counts is None else stage_counts) + assert len(stage_counts) == len(processors) + route_counts = [len(expected) * copies, + len(expected if location == "input" else records) * copies] + received = [[], []] + lock = threading.Lock() + + class Sink(BaseHTTPRequestHandler): + def do_POST(self): + records = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with lock: + received[int(self.path[1:])].extend(records) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_args): + pass + + sink = ThreadingHTTPServer(("127.0.0.1", 0), Sink) + thread = threading.Thread(target=sink.serve_forever, daemon=True) + thread.start() + input_config = { + "name": "http", "listen": "127.0.0.1", + "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", "tag": "test", + "storage.type": storage, + } + outputs = [{ + "name": "http", "match": "*", "host": "127.0.0.1", + "port": sink.server_port, "uri": f"/{route}", "format": "json", + "json_date_key": False, "workers": 2, + } for route in range(2)] + if location == "input": + input_config["processors"] = {"logs": processors} + else: + outputs[0]["processors"] = {"logs": processors} + config = { + "service": {"flush": 0.1, "grace": 1, "log_level": "error", + "storage.path": str(tmp_path / "storage"), + "http_server": True, "http_listen": "127.0.0.1", + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [input_config], "outputs": outputs}, + } + config_path = tmp_path / "fluent-bit.yaml" + config_path.write_text(yaml.safe_dump(config)) + service = FluentBitTestService(str(config_path)) + try: + service.start() + def send(index): + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/test.{index}", + json=records, timeout=30) + assert response.status_code == 201, response.text + + with ThreadPoolExecutor(max_workers=min(copies, 4)) as pool: + list(pool.map(send, range(copies))) + service.wait_for_condition( + lambda: all(len(route_records) >= count + for route_records, count in zip(received, route_counts)), + timeout=30, interval=0.1, description="both output routes to finish", + ) + scope = "input" if location == "input" else "output" + + def metrics_ready(): + response = requests.get( + f"http://127.0.0.1:{service.flb.http_monitoring_port}/api/v2/metrics/prometheus", + timeout=10, + ) + response.raise_for_status() + values = processor_metrics(response.text, scope) + for stage, counts in enumerate(stage_counts): + for metric, count in zip(["items_in_total", "items_out_total"], counts): + if values.get(("fluentbit_processor_" + metric, stage), 0) != count * copies: + return None + return values + + values = service.wait_for_condition(metrics_ready, timeout=30, interval=0.2, + description="per-processor accounting") + invocations = values[("fluentbit_processor_invocations_total", 0)] + if location == "input": + assert invocations == total_records + elif copies == 1: + assert invocations == 1 + else: + assert 1 <= invocations <= total_records + for stage in range(len(processors)): + assert values[("fluentbit_processor_invocations_total", stage)] == invocations + assert values.get(("fluentbit_processor_errors_total", stage), 0) == 0 + before, after = stage_counts[stage] + assert values.get(("fluentbit_processor_items_drop_total", stage), 0) == ( + max(0, before - after) * copies + ) + assert values.get(("fluentbit_processor_items_add_total", stage), 0) == ( + max(0, after - before) * copies + ) + + finally: + try: + service.stop() + finally: + sink.shutdown() + sink.server_close() + thread.join(timeout=5) + assert sorted(received[0], key=lambda record: record["id"]) == sorted( + expected * copies, key=lambda record: record["id"] + ) + control = expected if location == "input" else records + assert sorted(received[1], key=lambda record: record["id"]) == sorted( + control * copies, key=lambda record: record["id"] + ) + + +@pytest.mark.parametrize("action", ["insert", "upsert", "delete", "rename"]) +@pytest.mark.parametrize("location", ["input", "output"]) +@pytest.mark.parametrize("storage", ["memory", "filesystem"]) +def test_body_actions(tmp_path, action, location, storage): + processor = {"name": "content_modifier", "context": "body", "action": action, + "key": "target"} + if action != "delete": + processor["value"] = "replacement" + expected = deepcopy(RECORDS) + for record in expected: + if action == "insert": + record.setdefault("target", "replacement") + elif action == "upsert": + record["target"] = "replacement" + elif action == "delete": + record.pop("target", None) + elif "target" in record: + record["replacement"] = record.pop("target") + run_pipeline(tmp_path, [processor], expected, location=location, storage=storage) + + +@pytest.mark.parametrize("location", ["input", "output"]) +def test_mixed_processor_chain_and_condition(tmp_path, location): + processors = [ + {"name": "content_modifier", "action": "insert", "key": "inserted", "value": "yes"}, + # A filter boundary consumes raw buffers and materializes them independently. + {"name": "modify", "add": "filtered yes"}, + {"name": "content_modifier", "action": "upsert", "key": "conditional", "value": "yes", + "condition": {"op": "and", "rules": [{"field": "$target", "op": "eq", "value": "original"}]}}, + # Later native units must see the materialized condition result. + {"name": "content_modifier", "action": "rename", "key": "target", "value": "renamed"}, + {"name": "content_modifier", "action": "delete", "key": "inserted"}, + ] + expected = deepcopy(RECORDS) + for record in expected: + record["filtered"] = "yes" + expected[0]["conditional"] = "yes" + expected[0]["renamed"] = expected[0].pop("target") + run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem") + + +@pytest.mark.parametrize("action,key", [("insert", "id"), ("delete", "missing"), + ("rename", "missing")]) +@pytest.mark.parametrize("location", ["input", "output"]) +def test_body_noop_preserves_shared_buffer(tmp_path, action, key, location): + processor = {"name": "content_modifier", "action": action, "key": key} + if action != "delete": + processor["value"] = "replacement" + run_pipeline(tmp_path, [processor], RECORDS, location=location, storage="filesystem") + + +@pytest.mark.parametrize("location", ["input", "output"]) +@pytest.mark.parametrize("mixed", [False, True], ids=["fused", "cfl-fallback"]) +def test_ordered_native_segment(tmp_path, location, mixed): + processors = [ + {"name": "content_modifier", "action": "insert", "key": "stage0", "value": "initial"}, + {"name": "content_modifier", "action": "rename", "key": "stage0", "value": "stage1"}, + {"name": "content_modifier", "action": "upsert", "key": "stage1", "value": "updated"}, + {"name": "content_modifier", "action": "delete", "key": "target"}, + {"name": "content_modifier", "action": "insert", "key": "target", "value": "done"}, + {"name": "content_modifier", "action": "delete", "key": "stage1"}, + ] + expected = deepcopy(RECORDS) + for record in expected: + record["target"] = "done" + if mixed: + processors.append({"name": "content_modifier", "action": "hash", "key": "target"}) + for record in expected: + record["target"] = hashlib.sha256(b"done").hexdigest() + run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem") + + +@pytest.mark.parametrize("location", ["input", "output"]) +def test_concurrent_fused_segments(tmp_path, location): + processors = [{"name": "content_modifier", "action": "upsert", "key": f"field{index}", + "value": f"value{index}"} for index in range(6)] + records = [{"id": index, "nested": {"values": [True, None, index]}} + for index in range(256)] + expected = deepcopy(records) + for record in expected: + record.update({f"field{index}": f"value{index}" for index in range(6)}) + run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem", + records=records, copies=8) + + +@pytest.mark.parametrize("location", ["input", "output"]) +@pytest.mark.parametrize("keep_id", [1, 99], ids=["partial-drop", "empty-batch"]) +def test_native_drop_accounting(tmp_path, location, keep_id): + # SQL and both modifiers share CFL, including when SQL removes every record. + processors = [ + {"name": "content_modifier", "action": "insert", "key": "before", "value": "yes"}, + {"name": "sql", "query": f"SELECT * FROM STREAM WHERE id = {keep_id};"}, + {"name": "content_modifier", "action": "insert", "key": "after", "value": "yes"}, + ] + expected = [dict(record, before="yes", after="yes") + for record in RECORDS if record["id"] == keep_id] + remaining = len(expected) + run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem", + stage_counts=[(2, 2), (2, remaining), (remaining, remaining)]) + + +@pytest.mark.parametrize("location", ["input", "output"]) +def test_multiple_fused_segments_and_noop_tail(tmp_path, location): + processors = [ + {"name": "content_modifier", "action": "insert", "key": "stage0", "value": "yes"}, + {"name": "content_modifier", "action": "rename", "key": "stage0", "value": "stage1"}, + {"name": "modify", "rename": "stage1 filtered"}, + {"name": "content_modifier", "action": "rename", "key": "filtered", "value": "stage2"}, + {"name": "content_modifier", "action": "upsert", "key": "stage2", "value": "done"}, + {"name": "modify", "add": "boundary yes"}, + # NOTOUCH must retain the buffer allocated by the preceding segment/filter. + {"name": "content_modifier", "action": "insert", "key": "id", "value": "unused"}, + {"name": "content_modifier", "action": "delete", "key": "missing"}, + ] + expected = [dict(record, stage2="done", boundary="yes") for record in RECORDS] + run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem") + + +@pytest.mark.parametrize("location", ["input", "output"]) +@pytest.mark.parametrize("matches", [True, False], ids=["condition-matches", "condition-false"]) +def test_condition_observes_previous_native_edit(tmp_path, location, matches): + processors = [ + {"name": "content_modifier", "action": "insert", "key": "gate", "value": "ready"}, + {"name": "content_modifier", "action": "upsert", "key": "conditional", "value": "yes", + "condition": {"op": "and", "rules": [ + {"field": "$gate", "op": "eq", "value": "ready" if matches else "absent"}]}}, + {"name": "content_modifier", "action": "rename", "key": "gate", "value": "renamed"}, + ] + expected = [dict(record, renamed="ready") for record in RECORDS] + if matches: + for record in expected: + record["conditional"] = "yes" + run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem") + + +@pytest.mark.parametrize("location", ["input", "output"]) +@pytest.mark.parametrize("depth", [4, 40], ids=["raw-compatible", "deep-cfl-fallback"]) +def test_nested_and_boundary_values(tmp_path, location, depth): + nested = {"values": [None, True, False, -9223372036854775808, 18446744073709551615, + 1.25, "embedded\0nul", "Unicode: café 日本語", [], {}]} + for _ in range(depth): + nested = {"nested": nested} + # On output, a deep second record forces fallback after the first was rewritten. + records = [{"id": 1, "target": "original", "payload": {}, + "large": "x" * 65536}, {"id": 2, "payload": nested}] + processors = [ + {"name": "content_modifier", "action": "upsert", "key": "target", "value": "updated"}, + {"name": "content_modifier", "action": "rename", "key": "target", "value": "renamed"}, + ] + expected = [dict(record, renamed="updated") for record in records] + for record in expected: + record.pop("target", None) + run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem", + records=records) + + +@pytest.mark.parametrize("location", ["input", "output"]) +def test_fused_mixed_noop_and_growing_maps(tmp_path, location): + records = [ + {"id": 1, "target": {"nested": [True, None, "preserved"]}}, + dict({"id": 2}, **{f"field{index}": f"value{index}" for index in range(128)}), + {"id": 3, "target": "last-record"}, + {"id": 4}, + ] + processors = [ + {"name": "content_modifier", "action": "insert", "key": "id", "value": "unused"}, + {"name": "content_modifier", "action": "rename", "key": "target", "value": "renamed"}, + {"name": "content_modifier", "action": "delete", "key": "missing"}, + ] + expected = deepcopy(records) + for record in expected: + if "target" in record: + record["renamed"] = record.pop("target") + run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem", + records=records) diff --git a/tests/integration/scenarios/processor_content_modifier/tests/test_grouped_logs.py b/tests/integration/scenarios/processor_content_modifier/tests/test_grouped_logs.py new file mode 100644 index 00000000000..e99111f4861 --- /dev/null +++ b/tests/integration/scenarios/processor_content_modifier/tests/test_grouped_logs.py @@ -0,0 +1,175 @@ +"""Grouped OTLP fallback must preserve envelope data and logical-record accounting.""" + +from copy import deepcopy +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import threading + +import pytest +import requests +import yaml +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + +from utils.test_service import FluentBitTestService +from test_content_modifier import processor_metrics + + +def grouped_payload(): + payload = ExportLogsServiceRequest() + for group in range(2): + resource = payload.resource_logs.add() + resource.schema_url = f"https://schemas.example/resource/{group}" + attribute = resource.resource.attributes.add(key="service.name") + attribute.value.string_value = f"service-{group}" + scope = resource.scope_logs.add() + scope.schema_url = f"https://schemas.example/scope/{group}" + scope.scope.name = f"scope-{group}" + scope.scope.version = "1.2.3" + scope.scope.attributes.add(key="scope-key").value.string_value = "scope-value" + for index in range(2): + record_id = group * 2 + index + 1 + record = scope.log_records.add() + record.time_unix_nano = 1700000000000000000 + record_id + record.observed_time_unix_nano = 1700000001000000000 + record_id + record.severity_number = 9 + record.severity_text = "INFO" + record.trace_id = bytes([record_id]) * 16 + record.span_id = bytes([record_id]) * 8 + record.flags = record_id + record.attributes.add(key="original-attribute").value.string_value = "preserved" + record.body.kvlist_value.values.add(key="id").value.int_value = record_id + record.body.kvlist_value.values.add(key="target").value.string_value = "original" + return payload + + +def flatten(payloads): + """Associate every log with its complete resource and scope envelopes.""" + result = [] + for payload in payloads: + for resource in payload.resource_logs: + for scope in resource.scope_logs: + for record in scope.log_records: + result.append((resource.schema_url, resource.resource, + scope.schema_url, scope.scope, record)) + return sorted(result, key=lambda entry: next( + item.value.int_value for item in entry[4].body.kvlist_value.values if item.key == "id" + )) + + +@pytest.mark.parametrize("location", ["input", "output"]) +@pytest.mark.parametrize("mixed", [False, True], ids=["group-fallback", "mixed-native-fallback"]) +def test_grouped_fallback_envelopes_and_accounting(tmp_path, location, mixed): + source = grouped_payload() + expected = deepcopy(source) + for resource in expected.resource_logs: + for scope in resource.scope_logs: + for record in scope.log_records: + for item in record.body.kvlist_value.values: + if item.key == "target": + item.key = "renamed" + item.value.string_value = (hashlib.sha256(b"updated").hexdigest() + if mixed else "updated") + record.body.kvlist_value.values.add(key="after").value.string_value = "yes" + remaining = 4 + processors = [ + {"name": "content_modifier", "action": "upsert", "key": "target", "value": "updated"}, + {"name": "content_modifier", "action": "rename", "key": "target", "value": "renamed"}, + ] + counts = [(4, 4), (4, 4)] + if mixed: + processors.append({"name": "content_modifier", "action": "hash", "key": "renamed"}) + counts.append((4, 4)) + processors.append({"name": "content_modifier", "action": "insert", "key": "after", + "value": "yes"}) + counts.append((remaining, remaining)) + received = [[], []] + lock = threading.Lock() + + class Sink(BaseHTTPRequestHandler): + def do_POST(self): + payload = ExportLogsServiceRequest() + payload.ParseFromString(self.rfile.read(int(self.headers["Content-Length"]))) + with lock: + received[int(self.path[-1])].append(payload) + self.send_response(200) + self.send_header("Content-Type", "application/x-protobuf") + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *_args): + pass + + sink = ThreadingHTTPServer(("127.0.0.1", 0), Sink) + thread = threading.Thread(target=sink.serve_forever, daemon=True) + thread.start() + input_config = {"name": "opentelemetry", "listen": "127.0.0.1", + "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", "tag": "test", + "storage.type": "filesystem"} + outputs = [{"name": "opentelemetry", "match": "*", "host": "127.0.0.1", + "port": sink.server_port, "logs_uri": f"/logs/{route}"} + for route in range(2)] + if location == "input": + input_config["processors"] = {"logs": processors} + else: + outputs[0]["processors"] = {"logs": processors} + config = { + "service": {"flush": 0.1, "grace": 1, "log_level": "error", + "storage.path": str(tmp_path / "storage"), "http_server": True, + "http_listen": "127.0.0.1", + "http_port": "${FLUENT_BIT_HTTP_MONITORING_PORT}"}, + "pipeline": {"inputs": [input_config], "outputs": outputs}, + } + config_path = tmp_path / "fluent-bit.yaml" + config_path.write_text(yaml.safe_dump(config)) + service = FluentBitTestService(str(config_path)) + try: + service.start() + response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/v1/logs", + data=source.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, timeout=30) + assert response.status_code == 201, response.text + control_count = remaining if location == "input" else 4 + service.wait_for_condition( + lambda: len(flatten(received[0])) >= remaining and + len(flatten(received[1])) >= control_count, + timeout=30, interval=0.1, description="both grouped output routes to finish", + ) + for route, payloads in enumerate(received): + (tmp_path / f"route-{route}.txt").write_text("\n".join(str(p) for p in payloads)) + + def metrics_ready(): + response = requests.get( + f"http://127.0.0.1:{service.flb.http_monitoring_port}/api/v2/metrics/prometheus", + timeout=10, + ) + response.raise_for_status() + values = processor_metrics(response.text, location, owner="opentelemetry.0") + for stage, (before, after) in enumerate(counts): + if values.get(("fluentbit_processor_items_in_total", stage), 0) != before: + return None + if values.get(("fluentbit_processor_items_out_total", stage), 0) != after: + return None + return values + + values = service.wait_for_condition(metrics_ready, timeout=30, interval=0.2, + description="group markers excluded from accounting") + for stage, (before, after) in enumerate(counts): + assert values[("fluentbit_processor_invocations_total", stage)] == 1 + assert values.get(("fluentbit_processor_errors_total", stage), 0) == 0 + assert values.get(("fluentbit_processor_items_drop_total", stage), 0) == before - after + assert values.get(("fluentbit_processor_items_add_total", stage), 0) == 0 + finally: + try: + service.stop() + finally: + sink.shutdown() + sink.server_close() + thread.join(timeout=5) + assert flatten(received[0]) == flatten([expected]) + assert flatten(received[1]) == flatten([expected if location == "input" else source]) + # Group envelopes must remain paired with their logical records. + for route in received: + for payload in route: + for resource in payload.resource_logs: + assert resource.scope_logs + assert all(scope.log_records for scope in resource.scope_logs) From 189f827290d09e1c6f46aa004db6a334f4cf8879 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 1 Oct 2026 22:05:16 -0600 Subject: [PATCH 06/13] benchmarks: compare chained log processor pipelines Signed-off-by: Eduardo Silva --- benchmarks/pipeline_compare.py | 247 +++++++++++++++++++++++++++++++++ 1 file changed, 247 insertions(+) create mode 100644 benchmarks/pipeline_compare.py diff --git a/benchmarks/pipeline_compare.py b/benchmarks/pipeline_compare.py new file mode 100644 index 00000000000..18a9f67269e --- /dev/null +++ b/benchmarks/pipeline_compare.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Measure completed HTTP/Forward -> body content_modifier -> null pipelines. + +Linux only. Uses equal CPU affinity, fresh processes, alternating run order, +10,000 warm-up records, four clients, and exact output-counter checks. CPU +seconds count only Fluent Bit; wall time ends after output completion. The +Forward workload uses PackedForward with V2 events and no compression/acks. +Field counts represent body width. Compare each input and width separately. +""" + +import argparse +import concurrent.futures +import csv +import datetime +import hashlib +import http.client +import json +import os +from pathlib import Path +import platform +import socket +import struct +import subprocess +import tempfile +import time + + +WARMUP_RECORDS = 10000 +BATCH_RECORDS = 1000 +CLIENTS = 4 + + +def free_port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def get_metrics(port): + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=2) + try: + connection.request("GET", "/api/v1/metrics") + response = connection.getresponse() + if response.status != 200: + raise RuntimeError(f"Metrics HTTP status: {response.status}") + return json.loads(response.read()) + finally: + connection.close() + + +def wait_for_count(port, expected, process): + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("Fluent Bit exited before output completion") + metrics = get_metrics(port) + output = metrics.get("output", {}).get("null.0", {}) + count = output.get("proc_records", 0) + if count >= expected: + if count != expected or output.get("dropped_records", 0) != 0: + raise RuntimeError(f"Unexpected output counters: {output}") + return + time.sleep(0.01) + raise TimeoutError(f"Output did not reach {expected} records") + + +def cpu_seconds(pid): + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + return (int(fields[11]) + int(fields[12])) / os.sysconf("SC_CLK_TCK") + + +def send_http(port, payload, batches): + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30) + try: + for _ in range(batches): + connection.request("POST", "/bench", payload, {"Content-Type": "application/json"}) + response = connection.getresponse() + body = response.read() + if response.status != 201: + raise RuntimeError(f"Input response: {response.status} {body!r}") + finally: + connection.close() + + +def pack_string(value): + value = value.encode() + return b"\xdb" + struct.pack(">I", len(value)) + value + + +def send_forward(port, payload, batches): + with socket.create_connection(("127.0.0.1", port), timeout=30) as connection: + for _ in range(batches): + connection.sendall(payload) + + +def make_payload(input_name, fields): + record = {f"field{index}": "representative log attribute" for index in range(fields)} + if input_name == "http": + return json.dumps([record] * BATCH_RECORDS, separators=(",", ":")).encode() + body = b"\xde" + struct.pack(">H", len(record)) + body += b"".join(pack_string(key) + pack_string(value) for key, value in record.items()) + entry = b"\x92\x92\xce" + struct.pack(">I", 1700000000) + b"\x80" + body + entries = entry * BATCH_RECORDS + return b"\x92" + pack_string("bench") + b"\xc6" + struct.pack(">I", len(entries)) + entries + + +def run(binary, records, fields, cpu, input_name, modifiers, mixed): + input_port = free_port() + metrics_port = free_port() + payload = make_payload(input_name, fields) + sender = send_http if input_name == "http" else send_forward + actions = "".join( + " - name: content_modifier\n" + " context: body\n" + " action: upsert\n" + f" key: environment{index}\n" + " value: production\n" + for index in range(modifiers) + ) + if mixed: + actions += (" - name: content_modifier\n" + " context: body\n" + " action: hash\n" + " key: environment0\n") + config = f"""service: + flush: 0.1 + grace: 1 + log_level: error + http_server: on + http_listen: 127.0.0.1 + http_port: {metrics_port} +pipeline: + inputs: + - name: {input_name} + listen: 127.0.0.1 + port: {input_port} + tag: bench + buffer_max_size: 16M + processors: + logs: +{actions} outputs: + - name: null + match: '*' +""" + with tempfile.TemporaryDirectory(prefix="flb-pipeline-perf-") as directory: + path = Path(directory) + config_path = path / "fluent-bit.yaml" + config_path.write_text(config) + with (path / "stderr.log").open("w+") as log: + process = subprocess.Popen( + ["taskset", "-c", str(cpu), str(binary), "-c", str(config_path)], + stdout=log, stderr=log, + ) + try: + deadline = time.monotonic() + 15 + while True: + try: + get_metrics(metrics_port) + break + except (OSError, ValueError, RuntimeError): + if time.monotonic() > deadline or process.poll() is not None: + log.seek(0) + raise RuntimeError(log.read()) + time.sleep(0.05) + sender(input_port, payload, WARMUP_RECORDS // BATCH_RECORDS) + wait_for_count(metrics_port, WARMUP_RECORDS, process) + start_cpu = cpu_seconds(process.pid) + start = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=CLIENTS) as pool: + tasks = [pool.submit(sender, input_port, payload, + records // (CLIENTS * BATCH_RECORDS)) + for _ in range(CLIENTS)] + for task in tasks: + task.result() + wait_for_count(metrics_port, records + WARMUP_RECORDS, process) + elapsed = time.monotonic() - start + cpu_used = cpu_seconds(process.pid) - start_cpu + return {"input": input_name, "records": records, "fields": fields, + "modifiers": modifiers, "mixed": mixed, + "elapsed_s": elapsed, "cpu_s": cpu_used, + "records_per_second": records / elapsed, + "cpu_s_per_million": cpu_used * 1e6 / records, + "payload_sha256": hashlib.sha256(payload).hexdigest(), + "output_records": records} + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--baseline", type=Path, required=True) + parser.add_argument("--candidate", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--records", type=int, default=1000000) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--input", choices=("http", "forward"), default="http") + parser.add_argument("--fields", type=int, nargs="+", default=[12, 128]) + parser.add_argument("--modifiers", type=int, default=1) + parser.add_argument("--mixed", action="store_true", help="Append a CFL-only hash operation") + args = parser.parse_args() + if args.records <= 0 or args.records % (CLIENTS * BATCH_RECORDS) or args.runs <= 0: + parser.error("records must be a positive multiple of 4000; runs must be positive") + if args.modifiers < 1: + parser.error("modifiers must be positive") + if any(fields < 1 or fields > 128 for fields in args.fields): + parser.error("fields must be between 1 and 128 (16M maximum batch size)") + cpu = min(os.sched_getaffinity(0)) + binaries = {"baseline": args.baseline.resolve(), "candidate": args.candidate.resolve()} + hashes = {name: hashlib.sha256(binary.read_bytes()).hexdigest() + for name, binary in binaries.items()} + manifest = {"created_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "platform": platform.platform(), "cpu": cpu, + "cpu_info": Path("/proc/cpuinfo").read_text(), + "arguments": {key: str(value) if isinstance(value, Path) else value + for key, value in vars(args).items()}, + "binaries": {name: str(binary) for name, binary in binaries.items()}, + "binary_sha256": hashes, "warmup_records": WARMUP_RECORDS, + "batch_records": BATCH_RECORDS, "clients": CLIENTS, + "source_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest()} + args.output.with_suffix(".manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + with args.output.open("w") as output: + writer = None + for fields in args.fields: + for repetition in range(1, args.runs + 1): + order = ("baseline", "candidate") if repetition % 2 else ("candidate", "baseline") + for system in order: + if hashlib.sha256(binaries[system].read_bytes()).hexdigest() != hashes[system]: + raise RuntimeError(f"The {system} binary changed during the campaign") + row = run(binaries[system], args.records, fields, cpu, args.input, + args.modifiers, args.mixed) + row.update(system=system, repetition=repetition, cpu=cpu, + binary_sha256=hashes[system]) + if writer is None: + writer = csv.DictWriter(output, fieldnames=list(row)) + writer.writeheader() + writer.writerow(row) + output.flush() + print(json.dumps(row), flush=True) + + +if __name__ == "__main__": + main() From 1befb788c7465236a7f0a4b07bedb454a9d679b7 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 2 Oct 2026 07:09:25 -0600 Subject: [PATCH 07/13] processor_content_modifier: report scratch allocation failures Signed-off-by: Eduardo Silva --- plugins/processor_content_modifier/cm_msgpack.c | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/processor_content_modifier/cm_msgpack.c b/plugins/processor_content_modifier/cm_msgpack.c index 2c9a014c7d3..e09a1f0211d 100644 --- a/plugins/processor_content_modifier/cm_msgpack.c +++ b/plugins/processor_content_modifier/cm_msgpack.c @@ -146,6 +146,7 @@ int cm_logs_process_raw(struct flb_processor_instance **instances, size_t instan if (required > capacity) { resized = flb_realloc(pairs, required * sizeof(msgpack_object_kv)); if (resized == NULL) { + flb_errno(); result = FLB_PROCESSOR_FAILURE; goto cleanup; } From 01c8282f8ac9131a9acae06debc45050415da3be Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 2 Oct 2026 07:09:25 -0600 Subject: [PATCH 08/13] tests: integration: stress processor input and output workers Signed-off-by: Eduardo Silva --- .../tests/test_content_modifier.py | 67 +++++++++++++++---- 1 file changed, 54 insertions(+), 13 deletions(-) diff --git a/tests/integration/scenarios/processor_content_modifier/tests/test_content_modifier.py b/tests/integration/scenarios/processor_content_modifier/tests/test_content_modifier.py index f7bfb8fba05..118a76ba906 100644 --- a/tests/integration/scenarios/processor_content_modifier/tests/test_content_modifier.py +++ b/tests/integration/scenarios/processor_content_modifier/tests/test_content_modifier.py @@ -35,7 +35,8 @@ def processor_metrics(text, scope, owner="http.0"): def run_pipeline(tmp_path, processors, expected, *, location="input", storage="memory", - records=None, copies=1, stage_counts=None): + records=None, copies=1, stage_counts=None, input_workers=1, + output_workers=2, input_threaded=False, clients=4, unique_copies=False): records = RECORDS if records is None else records total_records = len(records) * copies stage_counts = ([(len(records), len(records))] * len(processors) @@ -47,6 +48,8 @@ def run_pipeline(tmp_path, processors, expected, *, location="input", storage="m lock = threading.Lock() class Sink(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + def do_POST(self): records = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) with lock: @@ -58,18 +61,21 @@ def do_POST(self): def log_message(self, *_args): pass - sink = ThreadingHTTPServer(("127.0.0.1", 0), Sink) + class SinkServer(ThreadingHTTPServer): + request_queue_size = 128 + + sink = SinkServer(("127.0.0.1", 0), Sink) thread = threading.Thread(target=sink.serve_forever, daemon=True) thread.start() input_config = { "name": "http", "listen": "127.0.0.1", "port": "${FLUENT_BIT_TEST_LISTENER_PORT}", "tag": "test", - "storage.type": storage, + "storage.type": storage, "workers": input_workers, "threaded": input_threaded, } outputs = [{ "name": "http", "match": "*", "host": "127.0.0.1", "port": sink.server_port, "uri": f"/{route}", "format": "json", - "json_date_key": False, "workers": 2, + "json_date_key": False, "workers": output_workers, } for route in range(2)] if location == "input": input_config["processors"] = {"logs": processors} @@ -88,11 +94,16 @@ def log_message(self, *_args): try: service.start() def send(index): + batch = deepcopy(records) + if unique_copies: + for record in batch: + record["id"] += index * len(records) + record["request"] = index response = requests.post(f"http://127.0.0.1:{service.flb_listener_port}/test.{index}", - json=records, timeout=30) + json=batch, timeout=30) assert response.status_code == 201, response.text - with ThreadPoolExecutor(max_workers=min(copies, 4)) as pool: + with ThreadPoolExecutor(max_workers=min(copies, clients)) as pool: list(pool.map(send, range(copies))) service.wait_for_condition( lambda: all(len(route_records) >= count @@ -106,6 +117,8 @@ def metrics_ready(): f"http://127.0.0.1:{service.flb.http_monitoring_port}/api/v2/metrics/prometheus", timeout=10, ) + if response.status_code == 404: + return None response.raise_for_status() values = processor_metrics(response.text, scope) for stage, counts in enumerate(stage_counts): @@ -117,7 +130,7 @@ def metrics_ready(): values = service.wait_for_condition(metrics_ready, timeout=30, interval=0.2, description="per-processor accounting") invocations = values[("fluentbit_processor_invocations_total", 0)] - if location == "input": + if location == "input" and input_workers == 1: assert invocations == total_records elif copies == 1: assert invocations == 1 @@ -141,13 +154,20 @@ def metrics_ready(): sink.shutdown() sink.server_close() thread.join(timeout=5) - assert sorted(received[0], key=lambda record: record["id"]) == sorted( - expected * copies, key=lambda record: record["id"] - ) + def copied_records(source): + result = [] + for index in range(copies): + batch = deepcopy(source) + if unique_copies: + for record in batch: + record["id"] += index * len(records) + record["request"] = index + result.extend(batch) + return sorted(result, key=lambda record: record["id"]) + + assert sorted(received[0], key=lambda record: record["id"]) == copied_records(expected) control = expected if location == "input" else records - assert sorted(received[1], key=lambda record: record["id"]) == sorted( - control * copies, key=lambda record: record["id"] - ) + assert sorted(received[1], key=lambda record: record["id"]) == copied_records(control) @pytest.mark.parametrize("action", ["insert", "upsert", "delete", "rename"]) @@ -325,3 +345,24 @@ def test_fused_mixed_noop_and_growing_maps(tmp_path, location): record["renamed"] = record.pop("target") run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem", records=records) + + +@pytest.mark.parametrize("location", ["input", "output"]) +@pytest.mark.parametrize("threaded", [False, True], ids=["engine-input", "threaded-input"]) +@pytest.mark.parametrize("chain", ["fused", "filter-boundary"]) +def test_worker_threads_preserve_independent_batches(tmp_path, location, threaded, chain): + records = [dict({"id": index, "target": f"record-{index}"}, + **{f"field{field}": "payload" * 16 for field in range(32)}) + for index in range(128)] + processors = [{"name": "content_modifier", "action": "upsert", "key": f"stage{index}", + "value": f"value{index}"} for index in range(6)] + expected = deepcopy(records) + for record in expected: + record.update({f"stage{index}": f"value{index}" for index in range(6)}) + if chain == "filter-boundary": + processors.insert(3, {"name": "modify", "add": "boundary yes"}) + for record in expected: + record["boundary"] = "yes" + run_pipeline(tmp_path, processors, expected, location=location, storage="filesystem", + records=records, copies=32, clients=8, unique_copies=True, + input_workers=4, output_workers=4, input_threaded=threaded) From 9916eff03cfd06e9b78dd06f48e5b2b64a856f1c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 2 Oct 2026 07:09:25 -0600 Subject: [PATCH 09/13] benchmarks: compare OTLP JSON and protobuf processor pipelines Signed-off-by: Eduardo Silva --- benchmarks/pipeline_compare.py | 56 +++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/benchmarks/pipeline_compare.py b/benchmarks/pipeline_compare.py index 18a9f67269e..a9ed1359053 100644 --- a/benchmarks/pipeline_compare.py +++ b/benchmarks/pipeline_compare.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 -"""Measure completed HTTP/Forward -> body content_modifier -> null pipelines. +"""Measure completed HTTP/Forward/OTLP -> body content_modifier -> null pipelines. + +OTLP protobuf requires the Python dependencies in tests/integration/requirements.txt. +OTLP logs include resource/scope groups, which retain the CFL processing path. Linux only. Uses equal CPU affinity, fresh processes, alternating run order, 10,000 warm-up records, four clients, and exact output-counter checks. CPU @@ -69,11 +72,11 @@ def cpu_seconds(pid): return (int(fields[11]) + int(fields[12])) / os.sysconf("SC_CLK_TCK") -def send_http(port, payload, batches): +def send_http(port, payload, batches, path="/bench", content_type="application/json"): connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30) try: for _ in range(batches): - connection.request("POST", "/bench", payload, {"Content-Type": "application/json"}) + connection.request("POST", path, payload, {"Content-Type": content_type}) response = connection.getresponse() body = response.read() if response.status != 201: @@ -97,6 +100,19 @@ def make_payload(input_name, fields): record = {f"field{index}": "representative log attribute" for index in range(fields)} if input_name == "http": return json.dumps([record] * BATCH_RECORDS, separators=(",", ":")).encode() + if input_name.startswith("otlp-"): + log = {"timeUnixNano": "1700000000000000000", + "body": {"kvlistValue": {"values": [ + {"key": key, "value": {"stringValue": value}} for key, value in record.items() + ]}}} + request = {"resourceLogs": [{"resource": {"attributes": [ + {"key": "service.name", "value": {"stringValue": "perf"}}]}, + "scopeLogs": [{"scope": {"name": "benchmark"}, "logRecords": [log] * BATCH_RECORDS}]}]} + if input_name == "otlp-json": + return json.dumps(request, separators=(",", ":")).encode() + from google.protobuf.json_format import ParseDict + from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + return ParseDict(request, ExportLogsServiceRequest()).SerializeToString() body = b"\xde" + struct.pack(">H", len(record)) body += b"".join(pack_string(key) + pack_string(value) for key, value in record.items()) entry = b"\x92\x92\xce" + struct.pack(">I", 1700000000) + b"\x80" + body @@ -104,11 +120,17 @@ def make_payload(input_name, fields): return b"\x92" + pack_string("bench") + b"\xc6" + struct.pack(">I", len(entries)) + entries -def run(binary, records, fields, cpu, input_name, modifiers, mixed): +def run(binary, records, fields, cpu, input_name, modifiers, mixed, location): input_port = free_port() metrics_port = free_port() payload = make_payload(input_name, fields) - sender = send_http if input_name == "http" else send_forward + if input_name.startswith("otlp-"): + content_type = "application/x-protobuf" if input_name == "otlp-protobuf" else "application/json" + def sender(port, data, batches): + return send_http(port, data, batches, "/v1/logs", content_type) + else: + sender = send_http if input_name == "http" else send_forward + plugin_name = "opentelemetry" if input_name.startswith("otlp-") else input_name actions = "".join( " - name: content_modifier\n" " context: body\n" @@ -122,6 +144,11 @@ def run(binary, records, fields, cpu, input_name, modifiers, mixed): " context: body\n" " action: hash\n" " key: environment0\n") + processor_config = "" + if actions: + processor_config = " processors:\n logs:\n" + actions + input_processors = processor_config if location == "input" else "" + output_processors = processor_config if location == "output" else "" config = f"""service: flush: 0.1 grace: 1 @@ -131,17 +158,15 @@ def run(binary, records, fields, cpu, input_name, modifiers, mixed): http_port: {metrics_port} pipeline: inputs: - - name: {input_name} + - name: {plugin_name} listen: 127.0.0.1 port: {input_port} tag: bench buffer_max_size: 16M - processors: - logs: -{actions} outputs: +{input_processors} outputs: - name: null match: '*' -""" +{output_processors}""" with tempfile.TemporaryDirectory(prefix="flb-pipeline-perf-") as directory: path = Path(directory) config_path = path / "fluent-bit.yaml" @@ -176,7 +201,7 @@ def run(binary, records, fields, cpu, input_name, modifiers, mixed): elapsed = time.monotonic() - start cpu_used = cpu_seconds(process.pid) - start_cpu return {"input": input_name, "records": records, "fields": fields, - "modifiers": modifiers, "mixed": mixed, + "modifiers": modifiers, "mixed": mixed, "location": location, "elapsed_s": elapsed, "cpu_s": cpu_used, "records_per_second": records / elapsed, "cpu_s_per_million": cpu_used * 1e6 / records, @@ -198,15 +223,16 @@ def main(): parser.add_argument("--output", type=Path, required=True) parser.add_argument("--records", type=int, default=1000000) parser.add_argument("--runs", type=int, default=5) - parser.add_argument("--input", choices=("http", "forward"), default="http") + parser.add_argument("--input", choices=("http", "forward", "otlp-json", "otlp-protobuf"), default="http") parser.add_argument("--fields", type=int, nargs="+", default=[12, 128]) parser.add_argument("--modifiers", type=int, default=1) + parser.add_argument("--location", choices=("input", "output"), default="input") parser.add_argument("--mixed", action="store_true", help="Append a CFL-only hash operation") args = parser.parse_args() if args.records <= 0 or args.records % (CLIENTS * BATCH_RECORDS) or args.runs <= 0: parser.error("records must be a positive multiple of 4000; runs must be positive") - if args.modifiers < 1: - parser.error("modifiers must be positive") + if args.modifiers < 0 or (args.mixed and args.modifiers == 0): + parser.error("modifiers must be nonnegative; mixed requires at least one modifier") if any(fields < 1 or fields > 128 for fields in args.fields): parser.error("fields must be between 1 and 128 (16M maximum batch size)") cpu = min(os.sched_getaffinity(0)) @@ -232,7 +258,7 @@ def main(): if hashlib.sha256(binaries[system].read_bytes()).hexdigest() != hashes[system]: raise RuntimeError(f"The {system} binary changed during the campaign") row = run(binaries[system], args.records, fields, cpu, args.input, - args.modifiers, args.mixed) + args.modifiers, args.mixed, args.location) row.update(system=system, repetition=repetition, cpu=cpu, binary_sha256=hashes[system]) if writer is None: From 011217081b832bf33c1ed023662e40f33183b96b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Fri, 2 Oct 2026 07:17:44 -0600 Subject: [PATCH 10/13] benchmarks: wait for pipeline metrics snapshots Signed-off-by: Eduardo Silva --- benchmarks/pipeline_compare.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/benchmarks/pipeline_compare.py b/benchmarks/pipeline_compare.py index a9ed1359053..38115897f56 100644 --- a/benchmarks/pipeline_compare.py +++ b/benchmarks/pipeline_compare.py @@ -39,11 +39,17 @@ def free_port(): return listener.getsockname()[1] +class MetricsNotReady(RuntimeError): + pass + + def get_metrics(port): connection = http.client.HTTPConnection("127.0.0.1", port, timeout=2) try: connection.request("GET", "/api/v1/metrics") response = connection.getresponse() + if response.status == 404: + raise MetricsNotReady("Metrics snapshot is not available yet") if response.status != 200: raise RuntimeError(f"Metrics HTTP status: {response.status}") return json.loads(response.read()) @@ -56,7 +62,11 @@ def wait_for_count(port, expected, process): while time.monotonic() < deadline: if process.poll() is not None: raise RuntimeError("Fluent Bit exited before output completion") - metrics = get_metrics(port) + try: + metrics = get_metrics(port) + except MetricsNotReady: + time.sleep(0.01) + continue output = metrics.get("output", {}).get("null.0", {}) count = output.get("proc_records", 0) if count >= expected: From e76a2bb66e4d44630a8d29834db65ce0695fb36a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 26 Oct 2026 13:18:36 -0600 Subject: [PATCH 11/13] tests: runtime: verify raw processor count propagation Signed-off-by: Eduardo Silva --- tests/runtime/processor_content_modifier.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/runtime/processor_content_modifier.c b/tests/runtime/processor_content_modifier.c index 5ecc1c6e8c7..7705dd1e8bb 100644 --- a/tests/runtime/processor_content_modifier.c +++ b/tests/runtime/processor_content_modifier.c @@ -1884,6 +1884,9 @@ static void check_raw_chain_equivalence(const char *action, const char *json, int root_type; int raw_result; int expected_ret; + int input_records; + int actual_records; + int expected_records; char *data; char *saved; char *actual_json; @@ -1945,12 +1948,17 @@ static void check_raw_chain_equivalence(const char *action, const char *json, actual = NULL; expected = NULL; - TEST_CHECK(flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, - data, bytes, &actual, &actual_size) == 0); + input_records = flb_mp_count_log_records(data, bytes); + TEST_CHECK(flb_processor_run_with_count(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + data, bytes, &actual, &actual_size, + input_records, &actual_records) == 0); ins->p = &fallback; - TEST_CHECK(flb_processor_run(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, - data, bytes, &expected, &expected_size) == 0); + TEST_CHECK(flb_processor_run_with_count(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + data, bytes, &expected, &expected_size, + input_records, &expected_records) == 0); ins->p = original; + TEST_CHECK(actual_records == flb_mp_count_log_records(actual, actual_size)); + TEST_CHECK(actual_records == expected_records); TEST_CHECK(memcmp(data, saved, bytes) == 0); TEST_CHECK(flb_log_event_decoder_init(&actual_decoder, actual, actual_size) == 0); From aa14a9832e55b622dba48ba7d2f67042a7c7cb2d Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 26 Oct 2026 13:18:36 -0600 Subject: [PATCH 12/13] benchmarks: record processor CPU affinity and peak RSS Signed-off-by: Eduardo Silva --- benchmarks/pipeline_compare.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/benchmarks/pipeline_compare.py b/benchmarks/pipeline_compare.py index 38115897f56..ca7fafad21f 100644 --- a/benchmarks/pipeline_compare.py +++ b/benchmarks/pipeline_compare.py @@ -82,6 +82,13 @@ def cpu_seconds(pid): return (int(fields[11]) + int(fields[12])) / os.sysconf("SC_CLK_TCK") +def peak_rss_kib(pid): + for line in Path(f"/proc/{pid}/status").read_text().splitlines(): + if line.startswith("VmHWM:"): + return int(line.split()[1]) + raise RuntimeError("Process peak RSS is unavailable") + + def send_http(port, payload, batches, path="/bench", content_type="application/json"): connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30) try: @@ -215,6 +222,7 @@ def sender(port, data, batches): "elapsed_s": elapsed, "cpu_s": cpu_used, "records_per_second": records / elapsed, "cpu_s_per_million": cpu_used * 1e6 / records, + "peak_rss_kib": peak_rss_kib(process.pid), "payload_sha256": hashlib.sha256(payload).hexdigest(), "output_records": records} finally: @@ -233,6 +241,7 @@ def main(): parser.add_argument("--output", type=Path, required=True) parser.add_argument("--records", type=int, default=1000000) parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--cpu", type=int, help="CPU for Fluent Bit (default: first allowed CPU)") parser.add_argument("--input", choices=("http", "forward", "otlp-json", "otlp-protobuf"), default="http") parser.add_argument("--fields", type=int, nargs="+", default=[12, 128]) parser.add_argument("--modifiers", type=int, default=1) @@ -245,7 +254,10 @@ def main(): parser.error("modifiers must be nonnegative; mixed requires at least one modifier") if any(fields < 1 or fields > 128 for fields in args.fields): parser.error("fields must be between 1 and 128 (16M maximum batch size)") - cpu = min(os.sched_getaffinity(0)) + allowed_cpus = os.sched_getaffinity(0) + cpu = min(allowed_cpus) if args.cpu is None else args.cpu + if cpu not in allowed_cpus: + parser.error("cpu must be in the current process affinity mask") binaries = {"baseline": args.baseline.resolve(), "candidate": args.candidate.resolve()} hashes = {name: hashlib.sha256(binary.read_bytes()).hexdigest() for name, binary in binaries.items()} From d9662be1b9a77c2d58d3942c4ab0a9b6d228bf07 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 26 Oct 2026 13:19:14 -0600 Subject: [PATCH 13/13] benchmarks: document post-FTS processor fusion measurements Signed-off-by: Eduardo Silva --- benchmarks/results/post-fts-12505/README.md | 78 +++ .../post-fts-12505/confirm-forward-12-0.csv | 11 + .../post-fts-12505/confirm-forward-12-6.csv | 11 + .../post-fts-12505/confirm-forward-128-0.csv | 11 + .../post-fts-12505/confirm-forward-128-6.csv | 11 + .../post-fts-12505/forward-control.csv | 13 + .../results/post-fts-12505/forward-mixed.csv | 13 + .../results/post-fts-12505/forward-one.csv | 13 + .../results/post-fts-12505/forward-output.csv | 13 + .../results/post-fts-12505/forward-six.csv | 13 + .../results/post-fts-12505/http-six.csv | 13 + .../results/post-fts-12505/manifest.json | 468 ++++++++++++++++++ .../post-fts-12505/otlp-json-control.csv | 13 + .../results/post-fts-12505/otlp-json-six.csv | 13 + .../post-fts-12505/otlp-protobuf-control.csv | 13 + .../post-fts-12505/otlp-protobuf-six.csv | 13 + benchmarks/results/post-fts-12505/summary.csv | 25 + 17 files changed, 745 insertions(+) create mode 100644 benchmarks/results/post-fts-12505/README.md create mode 100644 benchmarks/results/post-fts-12505/confirm-forward-12-0.csv create mode 100644 benchmarks/results/post-fts-12505/confirm-forward-12-6.csv create mode 100644 benchmarks/results/post-fts-12505/confirm-forward-128-0.csv create mode 100644 benchmarks/results/post-fts-12505/confirm-forward-128-6.csv create mode 100644 benchmarks/results/post-fts-12505/forward-control.csv create mode 100644 benchmarks/results/post-fts-12505/forward-mixed.csv create mode 100644 benchmarks/results/post-fts-12505/forward-one.csv create mode 100644 benchmarks/results/post-fts-12505/forward-output.csv create mode 100644 benchmarks/results/post-fts-12505/forward-six.csv create mode 100644 benchmarks/results/post-fts-12505/http-six.csv create mode 100644 benchmarks/results/post-fts-12505/manifest.json create mode 100644 benchmarks/results/post-fts-12505/otlp-json-control.csv create mode 100644 benchmarks/results/post-fts-12505/otlp-json-six.csv create mode 100644 benchmarks/results/post-fts-12505/otlp-protobuf-control.csv create mode 100644 benchmarks/results/post-fts-12505/otlp-protobuf-six.csv create mode 100644 benchmarks/results/post-fts-12505/summary.csv diff --git a/benchmarks/results/post-fts-12505/README.md b/benchmarks/results/post-fts-12505/README.md new file mode 100644 index 00000000000..3a702dfa4c8 --- /dev/null +++ b/benchmarks/results/post-fts-12505/README.md @@ -0,0 +1,78 @@ +# Processor fusion after FTS + +These measurements compare post-FTS master +`05048e35d7106890006b7b593531bd51a16dfc93` with the reviewed raw-segment +fusion and live-record counting changes. The candidate also contains the +independent SQL envelope fix from PR #12506; these benchmark pipelines do +not invoke SQL. + +Both binaries use verified `-O3 -DNDEBUG` flags. Runs use Linux CPU 7, +fresh processes in alternating order, four clients, 1,000 records per batch, +10,000 warmup records, and null output. The host was shared. The confirmation +campaign ran after the competing benchmark and integration tests finished; +the CPU was not reserved. + +## Confirmation + +Each row contains five baseline/candidate pairs. Narrow runs measure 800,000 +records; wide runs measure 200,000. CPU values are median seconds per million +records, and throughput changes compare median completed-record rates. + +| Fields | Upserts | Master CPU | Candidate CPU | CPU reduction | Throughput change | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 12 | 0 | 0.8375 | 0.8250 | 1.5% | +0.1% | +| 12 | 6 | 3.7375 | 3.1500 | 15.7% | approximately 0% | +| 128 | 0 | 7.0000 | 6.8000 | 2.9% | -0.1% | +| 128 | 6 | 26.3000 | 21.2000 | 19.4% | +20.0% | + +Compatible plain-log fusion remains useful after FTS. CPU savings do not +always increase throughput because ingestion and flush timing can limit it. +These are workload-specific results, not general release performance claims. + +## Other workloads and limits + +The initial campaign contains 20 workload/width combinations with three pairs +each, covering Forward, HTTP, grouped OTLP JSON/protobuf, input/output +placement, one/six upserts, ingestion controls and a CFL-only hash tail. +Six-upsert HTTP CPU reductions were 19.7% / 28.4% for 12 / 128 fields; +Forward output-side reductions were 27.9% / 15.0%. + +Mixed Forward chains improved CPU by only 1.2% / 1.8%; no meaningful +mixed-chain speedup is claimed. Grouped OTLP declines raw fusion and measures +the live-record counting change instead. A resource group kept whole can +prevent the windowed path from splitting a batch. Initial ingestion controls +varied by about +/-6%, so not every observed grouped-fallback change can be +attributed to this patch. No general fallback speedup is claimed. + +Peak RSS is recorded, but varies with queued input and flush timing; no +memory-saving claim is made. Stress tests and Memcheck are not race-detector +proofs. + +All 160 runs delivered exactly 44 million measured records, plus 1.6 million +warmup records. Per-run CSVs retain output counts, payload/binary hashes, +CPU, elapsed time and peak RSS. `summary.csv` contains derived medians; +`manifest.json` records the base, machine and original invocation arguments. + +## Reproduce + +Configure both revisions with: + +```sh +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DFLB_DEBUG=Off \ + -DFLB_RELEASE=Off -DFLB_TESTS_RUNTIME=On -DFLB_TESTS_INTERNAL=On +cmake --build build -j8 --target fluent-bit-bin +``` + +Then run, substituting the two binaries: + +```sh +python benchmarks/pipeline_compare.py --baseline /path/to/master/fluent-bit \ + --candidate build/bin/fluent-bit --input forward --fields 12 --modifiers 6 \ + --records 800000 --runs 5 --cpu 7 --output narrow.csv +python benchmarks/pipeline_compare.py --baseline /path/to/master/fluent-bit \ + --candidate build/bin/fluent-bit --input forward --fields 128 --modifiers 6 \ + --records 200000 --runs 5 --cpu 7 --output wide.csv +``` + +Repeat with `--modifiers 0` for controls. OTLP protobuf requires the Python +dependencies in `tests/integration/requirements.txt`. diff --git a/benchmarks/results/post-fts-12505/confirm-forward-12-0.csv b/benchmarks/results/post-fts-12505/confirm-forward-12-0.csv new file mode 100644 index 00000000000..bba93a99154 --- /dev/null +++ b/benchmarks/results/post-fts-12505/confirm-forward-12-0.csv @@ -0,0 +1,11 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +forward,800000,12,0,False,input,0.9928723580669612,0.6699999999999999,805743.0479357211,0.8374999999999998,442452,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,0,False,input,1.006367997964844,0.6699999999999999,794937.837468821,0.8374999999999998,441076,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,800000,12,0,False,input,0.9997398369014263,0.6599999999999999,800208.1846408201,0.8249999999999998,441032,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,800000,12,0,False,input,0.9969168181996793,0.6499999999999999,802474.1737677882,0.8124999999999999,445340,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,0,False,input,1.00638108397834,0.6799999999999999,794927.5008603183,0.8499999999999999,442616,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,0,False,input,1.0002973289228976,0.6599999999999999,799762.2075642507,0.8249999999999998,442488,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,800000,12,0,False,input,0.9986403589136899,0.6499999999999999,801089.1937816646,0.8124999999999999,444516,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,4,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,800000,12,0,False,input,1.0004755570553243,0.6599999999999999,799619.7351933522,0.8249999999999998,444696,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,4,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,0,False,input,1.0019128548447043,0.69,798472.6377465228,0.8625,442416,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,5,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,0,False,input,0.9998237572144717,0.6399999999999999,800141.0190820185,0.7999999999999998,444636,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,5,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/confirm-forward-12-6.csv b/benchmarks/results/post-fts-12505/confirm-forward-12-6.csv new file mode 100644 index 00000000000..bc1a10011f5 --- /dev/null +++ b/benchmarks/results/post-fts-12505/confirm-forward-12-6.csv @@ -0,0 +1,11 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +forward,800000,12,6,False,input,2.9980473509058356,2.99,266840.3485216091,3.7375,493264,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,6,False,input,2.999171285890043,2.52,266740.35049738403,3.15,476652,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,800000,12,6,False,input,2.9987838559318334,2.45,266774.8122017985,3.0625,478436,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,800000,12,6,False,input,3.999303624033928,3.06,200034.82486110268,3.825,489888,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,6,False,input,2.9935032511129975,2.99,267245.40877066244,3.7375,491020,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,6,False,input,3.0000486311037093,2.52,266662.34397196496,3.15,479260,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,800000,12,6,False,input,2.996678540017456,2.53,266962.23479323875,3.1625,479140,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,4,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,800000,12,6,False,input,3.9969076770357788,3.06,200154.73577145592,3.825,491128,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,4,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,6,False,input,2.9968843879178166,2.99,266943.89787783107,3.7375,492352,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,baseline,5,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,800000,12,6,False,input,2.9998506139963865,2.54,266679.94608379644,3.175,479200,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,800000,candidate,5,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/confirm-forward-128-0.csv b/benchmarks/results/post-fts-12505/confirm-forward-128-0.csv new file mode 100644 index 00000000000..bd2cf15ade6 --- /dev/null +++ b/benchmarks/results/post-fts-12505/confirm-forward-128-0.csv @@ -0,0 +1,11 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +forward,200000,128,0,False,input,1.9964346389751881,1.4,100178.58641376019,7.0,1159352,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,0,False,input,1.9920537951402366,1.3699999999999999,100398.89509405562,6.85,890832,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,0,False,input,1.998302411986515,1.37,100084.95150700427,6.85,849496,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,0,False,input,1.9964519499335438,1.3699999999999999,100177.7177791118,6.85,926188,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,0,False,input,1.9910582008305937,1.42,100449.09782977092,7.1,1040044,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,0,False,input,2.0014939669054,1.3599999999999999,99925.35741150846,6.799999999999999,1005384,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,0,False,input,2.0006355331279337,1.3099999999999998,99968.23343795458,6.549999999999999,897848,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,4,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,0,False,input,2.0012926449999213,1.3699999999999999,99935.40949629976,6.85,897020,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,4,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,0,False,input,2.0019575799815357,1.42,99902.2167102285,7.1,863852,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,5,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,0,False,input,1.998365829931572,1.3599999999999999,100081.7753208122,6.799999999999999,1036880,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,5,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/confirm-forward-128-6.csv b/benchmarks/results/post-fts-12505/confirm-forward-128-6.csv new file mode 100644 index 00000000000..f80582e76ba --- /dev/null +++ b/benchmarks/results/post-fts-12505/confirm-forward-128-6.csv @@ -0,0 +1,11 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +forward,200000,128,6,False,input,6.000584039837122,5.2,33330.08898337648,26.0,807076,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,input,4.999083634000272,4.23,40007.33227180674,21.15,876840,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,input,4.994535424048081,4.26,40043.76443843492,21.3,766736,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,input,6.00052207079716,5.2,33330.43319236226,26.0,845596,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,input,6.002101794816554,5.26,33321.66078434742,26.3,824156,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,input,4.995816803071648,4.29,40033.49359748964,21.45,780272,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,input,5.001377354841679,4.16,39988.98419580082,20.8,771252,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,4,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,input,5.99883626610972,5.3,33339.799775815714,26.5,893208,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,4,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,input,5.999827126041055,5.3,33334.293771888835,26.5,901556,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,5,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,input,5.001570146996528,4.24,39987.442767368,21.2,799160,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,5,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/forward-control.csv b/benchmarks/results/post-fts-12505/forward-control.csv new file mode 100644 index 00000000000..6785f88d0c3 --- /dev/null +++ b/benchmarks/results/post-fts-12505/forward-control.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +forward,200000,12,0,False,input,1.0067021648865193,0.19,198668.49101545842,0.95,126068,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,0,False,input,1.0015343860723078,0.16,199693.5929322756,0.8,125228,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,0,False,input,0.9994890699163079,0.17,200102.23825333774,0.85,125280,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,0,False,input,1.0001376760192215,0.18,199972.4685865711,0.9,124820,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,0,False,input,0.9998822929337621,0.18,200023.54418456447,0.9,121896,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,0,False,input,1.005915999179706,0.17,198823.75880599767,0.85,125220,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,0,False,input,1.9956678620073944,1.49,100217.07710361423,7.45,1159780,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,0,False,input,1.994322617072612,1.4,100284.67725726952,7.0,1163124,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,0,False,input,1.9940772580448538,1.3499999999999999,100297.01667431649,6.749999999999999,876656,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,0,False,input,2.0039535481482744,1.44,99802.71258523295,7.2,1050052,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,0,False,input,1.9994846100453287,1.38,100025.77614011541,6.9,947960,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,0,False,input,1.9950553968083113,1.3299999999999998,100247.84290198653,6.649999999999999,886680,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/forward-mixed.csv b/benchmarks/results/post-fts-12505/forward-mixed.csv new file mode 100644 index 00000000000..01e8ca57ba8 --- /dev/null +++ b/benchmarks/results/post-fts-12505/forward-mixed.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +forward,200000,12,6,True,input,1.001058522844687,0.8400000000000001,199788.5192882272,4.2,152792,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,6,True,input,0.996151999104768,0.8400000000000001,200772.573040799,4.2,146208,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,6,True,input,0.9963867000769824,0.8300000000000001,200725.28064108815,4.15,146748,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,6,True,input,1.0023724848870188,0.8500000000000001,199526.62609503168,4.250000000000001,149072,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,6,True,input,0.998013166943565,0.8500000000000001,200398.1576841355,4.250000000000001,148780,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,6,True,input,1.0002943039871752,0.8600000000000001,199941.15652043562,4.300000000000001,146556,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,True,input,5.991122722160071,5.56,33382.7246202513,27.8,851728,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,True,input,5.995461818063632,5.4799999999999995,33358.564539169136,27.4,826964,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,True,input,5.993895425926894,5.42,33367.282174275184,27.1,764412,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,True,input,5.9899230760056525,5.5600000000000005,33389.41042517843,27.800000000000004,1009612,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,True,input,5.997193893883377,5.47,33348.930106125605,27.35,841756,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,True,input,5.990406497847289,5.46,33386.7159218447,27.3,842260,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/forward-one.csv b/benchmarks/results/post-fts-12505/forward-one.csv new file mode 100644 index 00000000000..8ab2c8bd3c8 --- /dev/null +++ b/benchmarks/results/post-fts-12505/forward-one.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +forward,200000,12,1,False,input,1.0058329470921308,0.59,198840.17577491494,2.95,115724,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,1,False,input,1.0061684199608862,0.47000000000000003,198773.8792356202,2.35,112384,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,1,False,input,0.9952117879875004,0.47000000000000003,200962.249858833,2.35,112428,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,1,False,input,0.9913082299754024,0.57,201753.5958568231,2.85,116224,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,1,False,input,1.0018683760426939,0.5599999999999999,199627.02165526498,2.8,116008,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,1,False,input,0.9980067599099129,0.48,200399.444206223,2.4,112272,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,1,False,input,4.99395498004742,4.84,40048.41869801976,24.2,955224,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,1,False,input,4.996709948172793,4.1899999999999995,40026.337745126955,20.95,968516,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,1,False,input,5.003507141955197,4.17,39971.96253063545,20.85,905320,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,1,False,input,5.000910534989089,4.9,39992.71704636411,24.5,977044,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,1,False,input,6.003885800018907,4.95,33311.759527366456,24.75,835736,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,1,False,input,4.995452326955274,4.1499999999999995,40036.414504610024,20.749999999999996,874836,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/forward-output.csv b/benchmarks/results/post-fts-12505/forward-output.csv new file mode 100644 index 00000000000..93ab8805fad --- /dev/null +++ b/benchmarks/results/post-fts-12505/forward-output.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +forward,200000,12,6,False,output,1.998002589913085,1.0899999999999999,100099.97034523373,5.449999999999998,131084,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,6,False,output,0.9972215790767223,0.8,200557.23241084497,4.0,130824,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,6,False,output,0.9935838328674436,0.8,201291.52003491033,4.0,131048,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,6,False,output,2.0009462600573897,1.1099999999999999,99952.70937174681,5.549999999999999,131212,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,6,False,output,2.0047704211901873,1.1199999999999999,99762.04650967689,5.6,132536,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,6,False,output,1.0066487649455667,0.8099999999999999,198679.02983104018,4.05,132008,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,output,7.994230131153017,7.9,25018.043853980693,39.5,1085088,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,output,6.995708146831021,6.590000000000001,28588.95708658141,32.95,1025624,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,output,7.000283502973616,6.61,28570.27146329759,33.05,1098640,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,output,7.994153988081962,7.75,25018.28214694999,38.75,1106128,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,output,8.003254666924477,7.74,24989.83330201297,38.7,1117316,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,output,7.000073621980846,6.57,28571.128076708,32.85,1016224,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/forward-six.csv b/benchmarks/results/post-fts-12505/forward-six.csv new file mode 100644 index 00000000000..01cef3e0e14 --- /dev/null +++ b/benchmarks/results/post-fts-12505/forward-six.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +forward,200000,12,6,False,input,1.0000438019633293,0.76,199991.23999103974,3.8,137956,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,6,False,input,0.9905856628902256,0.57,201900.76183463147,2.85,133224,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,6,False,input,0.9962736370507628,0.57,200748.0601334124,2.85,136492,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,12,6,False,input,1.0014044840354472,0.7899999999999999,199719.4971546787,3.9499999999999993,138364,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,6,False,input,0.9963762629777193,0.7699999999999999,200727.3832500688,3.849999999999999,138864,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,12,6,False,input,1.0074256260413677,0.6,198525.82149005952,3.0,133128,5be4ef982f1c40029ec7027e0d73ce8bfe65fdf0b2628eb414fa377f5887c065,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,input,6.000650831963867,5.65,33329.717992363985,28.25,910680,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,input,4.992494690930471,4.44,40060.13273550927,22.2,945284,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,input,5.001016900874674,4.39,39991.866447206,21.95,975236,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +forward,200000,128,6,False,input,5.994506567949429,5.52,33363.880368291095,27.6,953904,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,input,6.001924727810547,5.3,33322.64383011654,26.5,788640,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +forward,200000,128,6,False,input,5.000638610916212,4.31,39994.89176510522,21.55,831276,6fd8ced99b871b1d39bcfee36d15ea044f42b41408a99bf6f1b6d5c35945c9a5,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/http-six.csv b/benchmarks/results/post-fts-12505/http-six.csv new file mode 100644 index 00000000000..763b8fa0fbc --- /dev/null +++ b/benchmarks/results/post-fts-12505/http-six.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +http,200000,12,6,False,input,1.9999397890642285,1.52,100003.01063742522,7.6,140732,6b8914f1f7eee723649949a194f1e0faf24473a401edaa2980ef943b8f87e0dd,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +http,200000,12,6,False,input,1.9969044001773,1.22,100155.01993097042,6.1,110216,6b8914f1f7eee723649949a194f1e0faf24473a401edaa2980ef943b8f87e0dd,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +http,200000,12,6,False,input,2.0036986239720136,1.22,99815.41016559259,6.1,110772,6b8914f1f7eee723649949a194f1e0faf24473a401edaa2980ef943b8f87e0dd,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +http,200000,12,6,False,input,1.999299175105989,1.52,100035.05352789303,7.6,140180,6b8914f1f7eee723649949a194f1e0faf24473a401edaa2980ef943b8f87e0dd,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +http,200000,12,6,False,input,1.996228646952659,1.53,100188.92390173332,7.65,141780,6b8914f1f7eee723649949a194f1e0faf24473a401edaa2980ef943b8f87e0dd,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +http,200000,12,6,False,input,1.99685278814286,1.23,100157.60860669489,6.15,139568,6b8914f1f7eee723649949a194f1e0faf24473a401edaa2980ef943b8f87e0dd,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +http,200000,128,6,False,input,10.000861917855218,9.19,19998.276312857237,45.95,1018924,3df9f93e6226ed337e98adf8c852508a08abefccfc5a91e8f72a85db53b36a06,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +http,200000,128,6,False,input,6.995327848941088,6.949999999999999,28590.511312529095,34.74999999999999,1043164,3df9f93e6226ed337e98adf8c852508a08abefccfc5a91e8f72a85db53b36a06,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +http,200000,128,6,False,input,7.003100209869444,6.449999999999999,28558.78025537043,32.24999999999999,809436,3df9f93e6226ed337e98adf8c852508a08abefccfc5a91e8f72a85db53b36a06,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +http,200000,128,6,False,input,9.9996227780357,9.229999999999999,20000.754472388955,46.14999999999999,788324,3df9f93e6226ed337e98adf8c852508a08abefccfc5a91e8f72a85db53b36a06,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +http,200000,128,6,False,input,9.997235689079389,9.24,20005.530150546776,46.2,1049576,3df9f93e6226ed337e98adf8c852508a08abefccfc5a91e8f72a85db53b36a06,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +http,200000,128,6,False,input,7.006483386037871,6.609999999999999,28544.9902583868,33.05,790196,3df9f93e6226ed337e98adf8c852508a08abefccfc5a91e8f72a85db53b36a06,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/manifest.json b/benchmarks/results/post-fts-12505/manifest.json new file mode 100644 index 00000000000..fd92060e243 --- /dev/null +++ b/benchmarks/results/post-fts-12505/manifest.json @@ -0,0 +1,468 @@ +{ + "base_commit": "05048e35d7106890006b7b593531bd51a16dfc93", + "cpu_model": "AMD Ryzen 9 7940HS w/ Radeon 780M Graphics", + "candidate_scope": "Reviewed processor fusion/counting changes plus the independent SQL group fix from PR #12506. None of these benchmark pipelines invokes SQL.", + "measured_runs": 160, + "measured_records": 44000000, + "warmup_records": 1600000, + "campaigns": { + "confirm-forward-128-6": { + "created_utc": "2026-10-26T19:01:49.507948+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/confirm-forward-128-6.csv", + "records": 200000, + "runs": 5, + "cpu": 7, + "input": "forward", + "fields": [ + 128 + ], + "modifiers": 6, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "confirm-forward-128-0": { + "created_utc": "2026-10-26T19:00:59.459425+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/confirm-forward-128-0.csv", + "records": 200000, + "runs": 5, + "cpu": 7, + "input": "forward", + "fields": [ + 128 + ], + "modifiers": 0, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "forward-control": { + "created_utc": "2026-10-26T18:51:22.466685+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/forward-control.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "forward", + "fields": [ + 12, + 128 + ], + "modifiers": 0, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "confirm-forward-12-0": { + "created_utc": "2026-10-26T18:59:17.521927+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/confirm-forward-12-0.csv", + "records": 800000, + "runs": 5, + "cpu": 7, + "input": "forward", + "fields": [ + 12 + ], + "modifiers": 0, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "http-six": { + "created_utc": "2026-10-26T18:49:43.512721+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/http-six.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "http", + "fields": [ + 12, + 128 + ], + "modifiers": 6, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "otlp-json-six": { + "created_utc": "2026-10-26T18:52:16.511363+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/otlp-json-six.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "otlp-json", + "fields": [ + 12, + 128 + ], + "modifiers": 6, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "otlp-protobuf-six": { + "created_utc": "2026-10-26T18:56:06.466021+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/otlp-protobuf-six.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "otlp-protobuf", + "fields": [ + 12, + 128 + ], + "modifiers": 6, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "confirm-forward-12-6": { + "created_utc": "2026-10-26T18:59:57.507929+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/confirm-forward-12-6.csv", + "records": 800000, + "runs": 5, + "cpu": 7, + "input": "forward", + "fields": [ + 12 + ], + "modifiers": 6, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "forward-one": { + "created_utc": "2026-10-26T18:45:43.006308+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/forward-one.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "forward", + "fields": [ + 12, + 128 + ], + "modifiers": 1, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "otlp-protobuf-control": { + "created_utc": "2026-10-26T18:58:00.522779+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/otlp-protobuf-control.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "otlp-protobuf", + "fields": [ + 12, + 128 + ], + "modifiers": 0, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "otlp-json-control": { + "created_utc": "2026-10-26T18:54:25.458470+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/otlp-json-control.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "otlp-json", + "fields": [ + 12, + 128 + ], + "modifiers": 0, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "forward-mixed": { + "created_utc": "2026-10-26T18:46:55.453095+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/forward-mixed.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "forward", + "fields": [ + 12, + 128 + ], + "modifiers": 6, + "location": "input", + "mixed": true + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "forward-six": { + "created_utc": "2026-10-26T18:44:18.298234+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/forward-six.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "forward", + "fields": [ + 12, + 128 + ], + "modifiers": 6, + "location": "input", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + }, + "forward-output": { + "created_utc": "2026-10-26T18:48:13.506030+00:00", + "platform": "Linux-7.0.0-34-generic-x86_64-with-glibc2.39", + "cpu": 7, + "arguments": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "build/bin/fluent-bit", + "output": "benchmarks/post-fts-pr-review/forward-output.csv", + "records": 200000, + "runs": 3, + "cpu": 7, + "input": "forward", + "fields": [ + 12, + 128 + ], + "modifiers": 6, + "location": "output", + "mixed": false + }, + "binaries": { + "baseline": "/tmp/flb-post-fts-release-baseline/build/bin/fluent-bit", + "candidate": "/tmp/flb-post-fts-pr-review/build/bin/fluent-bit" + }, + "binary_sha256": { + "baseline": "666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce", + "candidate": "fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485" + }, + "warmup_records": 10000, + "batch_records": 1000, + "clients": 4, + "source_sha256": "d3fffa8c553383f1adc97d5fbba63869665a72f5291648a1f19d769e75b4c208" + } + } +} diff --git a/benchmarks/results/post-fts-12505/otlp-json-control.csv b/benchmarks/results/post-fts-12505/otlp-json-control.csv new file mode 100644 index 00000000000..6f25cf7a63f --- /dev/null +++ b/benchmarks/results/post-fts-12505/otlp-json-control.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +otlp-json,200000,12,0,False,input,1.0004364079795778,0.8200000000000001,199912.75647785366,4.1000000000000005,116052,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,12,0,False,input,1.0040348279289901,0.8700000000000001,199196.27729701117,4.3500000000000005,117600,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,12,0,False,input,2.001289489911869,1.22,99935.56704722784,6.1,116628,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,12,0,False,input,1.0019163880497217,0.8300000000000001,199617.45549377587,4.15,93088,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,12,0,False,input,0.9922501281835139,0.78,201562.08028527518,3.9,117700,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,12,0,False,input,0.9982183200772852,0.81,200356.9719943783,4.05,116276,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,128,0,False,input,8.99456052808091,8.45,22235.661139374446,42.25,835444,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,128,0,False,input,8.99909132020548,8.549999999999999,22224.466102587936,42.74999999999999,1036616,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,128,0,False,input,11.001722508808598,10.37,18178.971505586396,51.85,1018784,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,128,0,False,input,9.999627758050337,9.11,20000.74451161317,45.55,1039616,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,128,0,False,input,9.999739050865173,9.52,20000.5219118889,47.6,1006460,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,128,0,False,input,8.993671789066866,8.0,22237.858428759817,40.0,877660,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/otlp-json-six.csv b/benchmarks/results/post-fts-12505/otlp-json-six.csv new file mode 100644 index 00000000000..749d6ea4c82 --- /dev/null +++ b/benchmarks/results/post-fts-12505/otlp-json-six.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +otlp-json,200000,12,6,False,input,1.9992312639951706,1.5499999999999998,100038.45157979839,7.749999999999999,150892,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,12,6,False,input,1.997104758163914,1.4,100144.97195624068,7.0,148088,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,12,6,False,input,1.997214948059991,1.4,100139.4467802634,7.0,148888,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,12,6,False,input,2.0060883809346706,1.5499999999999998,99696.50485030805,7.749999999999999,117168,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,12,6,False,input,2.0018913438543677,1.5499999999999998,99905.5221523299,7.749999999999999,150228,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,12,6,False,input,1.9972194279544055,1.41,100139.22216090409,7.05,120664,d44638de7a308af41274b192c6942c282d989c6260cfc881c96b0546daa9b868,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,128,6,False,input,14.00050748907961,13.82,14285.196458485518,69.1,1060280,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,128,6,False,input,12.999571547145024,12.25,15385.12244612586,61.25,1059992,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,128,6,False,input,13.005646490026265,12.09,15377.936049036505,60.45,1065408,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-json,200000,128,6,False,input,14.005391475977376,13.8,14280.214897459184,69.0,1065416,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,128,6,False,input,14.993539287010208,14.0,13339.078663919723,70.0,1031288,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-json,200000,128,6,False,input,12.00076759699732,11.370000000000001,16665.60062791662,56.85000000000001,1018344,adc7f632a54f8b62479cdb111253cbe1a2f0e5065717b7d2045b07f5d3404f10,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/otlp-protobuf-control.csv b/benchmarks/results/post-fts-12505/otlp-protobuf-control.csv new file mode 100644 index 00000000000..61b040f3c73 --- /dev/null +++ b/benchmarks/results/post-fts-12505/otlp-protobuf-control.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +otlp-protobuf,200000,12,0,False,input,0.99972382793203,0.42,200055.2496720102,2.1,118976,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,12,0,False,input,1.0021191970445216,0.43,199577.05689088252,2.15,96784,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,12,0,False,input,0.9957805632147938,0.43,200847.46317433313,2.15,122680,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,12,0,False,input,1.0039975249674171,0.4,199203.67832230526,2.0,97280,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,12,0,False,input,0.9980884431861341,0.42,200383.04357232386,2.1,118612,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,12,0,False,input,1.003007770050317,0.42,199400.24990032407,2.1,97572,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,128,0,False,input,4.993398650083691,4.3,40052.8806160205,21.5,818256,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,128,0,False,input,5.000455466099083,4.33,39996.35660309609,21.65,998960,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,128,0,False,input,5.003482395084575,4.3500000000000005,39972.16022913964,21.750000000000004,979448,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,128,0,False,input,5.002415418159217,4.21,39980.68598501077,21.05,1010408,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,128,0,False,input,5.007386424113065,4.18,39940.995773144285,20.9,808616,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,128,0,False,input,4.995484042912722,4.33,40036.16031638564,21.65,919604,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/otlp-protobuf-six.csv b/benchmarks/results/post-fts-12505/otlp-protobuf-six.csv new file mode 100644 index 00000000000..6971bab7776 --- /dev/null +++ b/benchmarks/results/post-fts-12505/otlp-protobuf-six.csv @@ -0,0 +1,13 @@ +input,records,fields,modifiers,mixed,location,elapsed_s,cpu_s,records_per_second,cpu_s_per_million,peak_rss_kib,payload_sha256,output_records,system,repetition,cpu,binary_sha256 +otlp-protobuf,200000,12,6,False,input,2.0057717631570995,1.38,99712.24227685733,6.9,154496,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,12,6,False,input,2.009251391980797,1.2,99539.5602552413,6.0,153184,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,12,6,False,input,2.0070258378982544,1.18,99649.93784506472,5.9,120724,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,12,6,False,input,2.000750669045374,1.3599999999999999,99962.48063004612,6.799999999999999,121652,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,12,6,False,input,2.001366199925542,1.3699999999999999,99931.73663442538,6.85,153628,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,12,6,False,input,2.0027452639769763,1.18,99862.92495474312,5.9,122772,bfa1eaa82ea4032f179c9ec0e182d2dd93ceed58b5efa9466429b24b555fcef4,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,128,6,False,input,12.002283985028043,11.209999999999999,16663.49506889565,56.05,996644,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,baseline,1,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,128,6,False,input,9.99073393899016,9.52,20018.549309923423,47.6,1034684,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,candidate,1,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,128,6,False,input,9.998204312985763,9.469999999999999,20003.59201904267,47.34999999999999,1038096,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,candidate,2,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 +otlp-protobuf,200000,128,6,False,input,11.997836535098031,11.0,16669.67202086204,55.0,936516,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,baseline,2,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,128,6,False,input,11.992539214901626,11.12,16677.035314713423,55.6,1034752,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,baseline,3,7,666b24a4de01a05cb1cf7f0ce30c8857dfe53def76afa984c5be29e44cc397ce +otlp-protobuf,200000,128,6,False,input,10.000098504126072,9.479999999999999,19999.80299368845,47.39999999999999,956968,3a14d4e155df1cd848b0b6454ecd97bf4b80d1511fc128626af61cbc1219cda4,200000,candidate,3,7,fb6fa9629c86800cb970bcddb585b7a01e5c1704e7d771ef37370c893f96a485 diff --git a/benchmarks/results/post-fts-12505/summary.csv b/benchmarks/results/post-fts-12505/summary.csv new file mode 100644 index 00000000000..884acfc9eb6 --- /dev/null +++ b/benchmarks/results/post-fts-12505/summary.csv @@ -0,0 +1,25 @@ +case,fields,pairs,records_per_run,baseline_cpu_s_per_million,candidate_cpu_s_per_million,cpu_reduction_percent,throughput_change_percent,baseline_peak_rss_kib,candidate_peak_rss_kib +confirm-forward-12-0,12,5,800000,0.8374999999999998,0.8249999999999998,1.492537313432829,0.06519147361216948,442616,442488 +confirm-forward-12-6,12,5,800000,3.7375,3.15,15.719063545150503,-0.037474851452967695,491128,479140 +confirm-forward-128-0,128,5,200000,7.0,6.799999999999999,2.857142857142869,-0.09577225397682243,926188,897848 +confirm-forward-128-6,128,5,200000,26.3,21.2,19.391634980988602,20.03244014534593,845596,780272 +forward-control,12,3,200000,0.9,0.85,5.555555555555558,-0.13945702439271956,124820,125228 +forward-control,128,3,200000,7.2,6.749999999999999,6.250000000000011,0.25883439963658716,1050052,886680 +forward-mixed,12,3,200000,4.250000000000001,4.2,1.1764705882353121,0.46887646807649563,149072,146556 +forward-mixed,128,3,200000,27.8,27.3,1.7985611510791366,-0.046258794486608856,851728,826964 +forward-one,12,3,200000,2.85,2.35,17.543859649122805,0.3869328633735458,116008,112384 +forward-one,128,3,200000,24.5,20.85,14.897959183673459,0.08406705331840403,955224,905320 +forward-output,12,3,200000,5.549999999999999,4.0,27.92792792792791,100.6521220599705,131212,131048 +forward-output,128,3,200000,38.75,32.95,14.967741935483858,14.202086475925512,1106128,1025624 +forward-six,12,3,200000,3.849999999999999,2.85,25.97402597402596,0.37842664629039735,138364,133224 +forward-six,128,3,200000,27.6,21.95,20.471014492753625,19.997690272291702,910680,945284 +http-six,12,3,200000,7.6,6.1,19.736842105263165,0.1199243653565274,140732,110772 +http-six,128,3,200000,46.14999999999999,33.05,28.385698808234018,42.78851477725918,1018924,809436 +otlp-json-control,12,3,200000,4.1000000000000005,4.3500000000000005,-6.0975609756097615,-0.35839592903710304,116052,116628 +otlp-json-control,128,3,200000,45.55,42.74999999999999,6.147091108671798,11.118194073643561,1006460,1018784 +otlp-json-six,12,3,200000,7.749999999999999,7.0,9.677419354838701,0.23414584388721238,150228,148088 +otlp-json-six,128,3,200000,69.1,60.45,12.518089725036162,7.7373313819196765,1060280,1059992 +otlp-protobuf-control,12,3,200000,2.1,2.15,-2.3809523809523725,-0.23903035881920331,118612,97572 +otlp-protobuf-control,128,3,200000,21.05,21.65,-2.8503562945368044,0.039195470761033135,818256,979448 +otlp-protobuf-six,12,3,200000,6.85,5.9,13.868613138686126,-0.28199128610317814,153628,122772 +otlp-protobuf-six,128,3,200000,55.6,47.39999999999999,14.748201438848941,19.999913579632754,996644,1034684