From b430977a34887e3ba4e3e4b5b3b4cf5c5135e5d9 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 9 Nov 2026 11:50:04 +0100 Subject: [PATCH 1/5] decode_prometheus: start a new instance after _sum or _count A new instance of a summary or histogram was only detected once both _sum and _count of the previous instance had been seen. _sum is optional: summaries exported from Dropwizard timers, e.g. by the Jenkins Prometheus plugin, only have quantiles and _count. For a family with several label sets, the quantiles of the second instance were added to the first one with the _count sample type, and the decoding failed with "not enough samples for summary", dropping the whole payload. Also start a new instance when a quantile follows a _sum or _count line, and when a bucket follows the _sum or _count of an instance with buckets. Signed-off-by: Chris Burr --- src/cmt_decode_prometheus.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/cmt_decode_prometheus.c b/src/cmt_decode_prometheus.c index a6215a25..6b53ee71 100644 --- a/src/cmt_decode_prometheus.c +++ b/src/cmt_decode_prometheus.c @@ -1043,9 +1043,11 @@ static int parse_histogram_summary_name( } if (name_matched) { - if (sum_found && count_found) { + if ((sum_found && count_found) || is_previous_sum_or_count) { /* finish instance of the summary/histogram, the samples of the - * next instance using the base name are quantiles */ + * next instance using the base name are quantiles. _sum is + * optional (e.g. summaries of Dropwizard timers), so a quantile + * following a _sum or _count also starts a new instance */ return finish_duplicate_histogram_summary_sum_count( context, metric_name, @@ -1060,8 +1062,9 @@ static int parse_histogram_summary_name( /* invalid histogram/summary suffix, treat it as a different metric */ if (!strcmp(metric_name + parsed_name_len, "_bucket")) { - if (sum_found && count_found && has_buckets && is_previous_sum_or_count) { - /* already found both sum and count, so this is a new metric */ + if (has_buckets && is_previous_sum_or_count) { + /* a bucket following the _sum or _count of an instance with + * buckets starts a new instance, _sum being optional */ return finish_duplicate_histogram_summary_sum_count( context, metric_name, From 43b176e9cf14fa2fe16dfd1b28d4ef4dbd18d237 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 9 Nov 2026 11:50:13 +0100 Subject: [PATCH 2/5] decode_prometheus: count histogram buckets instead of assuming _sum The number of buckets of a histogram was computed as the number of samples minus 3, assuming that the "+Inf" bucket, _sum and _count are all present. Without _sum, which is optional, it was one too low: the last finite bucket was taken as the "+Inf" one, and a histogram with a single finite bucket failed with "cmt_histogram_buckets_create_size failed". Count the bucket samples instead, as is already done for summary quantiles, and report a syntax error when there are less than two. A histogram without _count was only rejected because of that sample arithmetic. Reject it explicitly, otherwise it would now be decoded with a count of zero. Signed-off-by: Chris Burr --- src/cmt_decode_prometheus.c | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/cmt_decode_prometheus.c b/src/cmt_decode_prometheus.c index 6b53ee71..7d462d44 100644 --- a/src/cmt_decode_prometheus.c +++ b/src/cmt_decode_prometheus.c @@ -433,6 +433,7 @@ static int add_metric_histogram(struct cmt_decode_prometheus_context *context) int ret = 0; int i; int has_le = CMT_FALSE; + int has_count = CMT_FALSE; size_t bucket_count; size_t bucket_index; size_t label_count = 0; @@ -458,11 +459,23 @@ static int add_metric_histogram(struct cmt_decode_prometheus_context *context) "not enough samples for histogram"); } - /* bucket_count = sample count - 3: - * - "Inf" bucket - * - sum - * - count */ - bucket_count = cfl_list_size(&context->metric.samples) - 3; + /* bucket_count = number of bucket samples without the "+Inf" one. Count + * them instead of assuming that sum and count are present, _sum is + * optional */ + bucket_count = 0; + cfl_list_foreach(head, &context->metric.samples) { + sample = cfl_list_entry(head, struct cmt_decode_prometheus_context_sample, _head); + if (sample->type == CMT_DECODE_PROMETHEUS_CONTEXT_SAMPLE_TYPE_BUCKET) { + bucket_count++; + } + } + if (bucket_count < 2) { + return report_error(context, + CMT_DECODE_PROMETHEUS_SYNTAX_ERROR, + "not enough buckets for histogram"); + } + bucket_count--; + if (context->opts.override_timestamp) { timestamp = context->opts.override_timestamp; } @@ -605,6 +618,7 @@ static int add_metric_histogram(struct cmt_decode_prometheus_context *context) } } bucket_defaults[bucket_index] = count; + has_count = CMT_TRUE; if (!timestamp) { ret = parse_timestamp(context, sample->value2, ×tamp); @@ -618,6 +632,13 @@ static int add_metric_histogram(struct cmt_decode_prometheus_context *context) } } + if (!has_count) { + ret = report_error(context, + CMT_DECODE_PROMETHEUS_SYNTAX_ERROR, + "missing histogram _count"); + goto end; + } + if (!timestamp) { /* No timestamp was specified, use default value */ timestamp = context->opts.default_timestamp; From 9d7a15164ff6a041fbf0c58ca7dafddfc5905b78 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 9 Nov 2026 11:50:13 +0100 Subject: [PATCH 3/5] tests: add prometheus decoder tests for instances without _sum Decode a summary family with several instances that have quantiles and _count but no _sum, as exported for Dropwizard timers by the Jenkins Prometheus plugin, and the same for a histogram family. Check that a histogram without _count is still rejected. Signed-off-by: Chris Burr --- tests/prometheus_parser.c | 112 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/tests/prometheus_parser.c b/tests/prometheus_parser.c index bb3881dd..b9185481 100644 --- a/tests/prometheus_parser.c +++ b/tests/prometheus_parser.c @@ -2000,6 +2000,116 @@ void test_duplicate_label_and_help() } } +/* + * Summaries exported from Dropwizard timers, e.g. by the Jenkins Prometheus + * plugin, have quantiles and a _count but no _sum. When a family has more than + * one instance, the quantiles of the next instance must start a new instance + * instead of being added to the previous one. + */ +void test_summary_instances_without_sum() +{ + int status; + char errbuf[256]; + struct cmt *cmt; + struct cmt_decode_prometheus_parse_opts opts; + cfl_sds_t result; + + memset(&opts, 0, sizeof(opts)); + errbuf[0] = '\0'; + opts.errbuf = errbuf; + opts.errbuf_size = sizeof(errbuf); + + cmt = NULL; + status = cmt_decode_prometheus_create(&cmt, + "# HELP builds Generated from Dropwizard metric import\n" + "# TYPE builds summary\n" + "builds{node=\"a\",quantile=\"0.5\",} 11\n" + "builds{node=\"a\",quantile=\"0.999\",} 12\n" + "builds_count{node=\"a\",} 1\n" + "builds{node=\"b\",quantile=\"0.5\",} 21\n" + "builds{node=\"b\",quantile=\"0.999\",} 22\n" + "builds_count{node=\"b\",} 2\n" + "builds{node=\"c\",quantile=\"0.5\",} 31\n" + "builds{node=\"c\",quantile=\"0.999\",} 32\n" + "builds_count{node=\"c\",} 3\n" + "# HELP other a gauge after the summary\n" + "# TYPE other gauge\n" + "other 5\n", 0, &opts); + TEST_CHECK(status == 0); + TEST_MSG("%s", errbuf); + if (status == 0) { + result = cmt_encode_prometheus_create(cmt, CMT_FALSE); + TEST_CHECK(strstr(result, + "builds{quantile=\"0.5\",node=\"a\"} 11\n") != NULL); + TEST_CHECK(strstr(result, + "builds{quantile=\"0.999\",node=\"a\"} 12\n") != NULL); + TEST_CHECK(strstr(result, "builds_count{node=\"a\"} 1\n") != NULL); + TEST_CHECK(strstr(result, + "builds{quantile=\"0.5\",node=\"b\"} 21\n") != NULL); + TEST_CHECK(strstr(result, + "builds{quantile=\"0.999\",node=\"b\"} 22\n") != NULL); + TEST_CHECK(strstr(result, "builds_count{node=\"b\"} 2\n") != NULL); + TEST_CHECK(strstr(result, + "builds{quantile=\"0.999\",node=\"c\"} 32\n") != NULL); + TEST_CHECK(strstr(result, "builds_count{node=\"c\"} 3\n") != NULL); + TEST_CHECK(strstr(result, "other 5\n") != NULL); + TEST_MSG("%s", result); + cfl_sds_destroy(result); + cmt_decode_prometheus_destroy(cmt); + } +} + +/* same as above for histograms, where OpenMetrics also makes _sum optional */ +void test_histogram_instances_without_sum() +{ + int status; + char errbuf[256]; + struct cmt *cmt; + struct cmt_decode_prometheus_parse_opts opts; + cfl_sds_t result; + + memset(&opts, 0, sizeof(opts)); + errbuf[0] = '\0'; + opts.errbuf = errbuf; + opts.errbuf_size = sizeof(errbuf); + + cmt = NULL; + status = cmt_decode_prometheus_create(&cmt, + "# TYPE h histogram\n" + "h_bucket{code=\"200\",le=\"1\"} 1\n" + "h_bucket{code=\"200\",le=\"+Inf\"} 2\n" + "h_count{code=\"200\"} 2\n" + "h_bucket{code=\"500\",le=\"1\"} 3\n" + "h_bucket{code=\"500\",le=\"+Inf\"} 4\n" + "h_count{code=\"500\"} 4\n", 0, &opts); + TEST_CHECK(status == 0); + TEST_MSG("%s", errbuf); + if (status == 0) { + result = cmt_encode_prometheus_create(cmt, CMT_FALSE); + TEST_CHECK(strstr(result, + "h_bucket{le=\"1.0\",code=\"200\"} 1\n") != NULL); + TEST_CHECK(strstr(result, "h_count{code=\"200\"} 2\n") != NULL); + TEST_CHECK(strstr(result, + "h_bucket{le=\"1.0\",code=\"500\"} 3\n") != NULL); + TEST_CHECK(strstr(result, + "h_bucket{le=\"+Inf\",code=\"500\"} 4\n") != NULL); + TEST_CHECK(strstr(result, "h_count{code=\"500\"} 4\n") != NULL); + TEST_MSG("%s", result); + cfl_sds_destroy(result); + cmt_decode_prometheus_destroy(cmt); + } + + /* _count is still required, decoding it as zero would change the total */ + cmt = NULL; + status = cmt_decode_prometheus_create(&cmt, + "# TYPE h histogram\n" + "h_bucket{le=\"1\"} 1\n" + "h_bucket{le=\"+Inf\"} 2\n" + "h_sum 5\n", 0, &opts); + TEST_CHECK(status == CMT_DECODE_PROMETHEUS_SYNTAX_ERROR); + TEST_MSG("%s", errbuf); +} + TEST_LIST = { {"header_help", test_header_help}, {"header_type", test_header_type}, @@ -2043,5 +2153,7 @@ TEST_LIST = { {"lexer_unmatched_input", test_lexer_unmatched_input}, {"histogram_summary_shorter_instance", test_histogram_summary_shorter_instance}, {"duplicate_label_and_help", test_duplicate_label_and_help}, + {"summary_instances_without_sum", test_summary_instances_without_sum}, + {"histogram_instances_without_sum", test_histogram_instances_without_sum}, { 0 } }; From b5e248618a6dfd499c62949401a07357c7e8b327 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 9 Nov 2026 13:50:09 +0100 Subject: [PATCH 4/5] decode_prometheus: keep histogram +Inf bucket when _count comes first The value of _count was written to the bucket slot at the current bucket index, assuming that it follows all the buckets of the instance. When _count comes before the buckets, as in the .NET exporter output of Fluent Bit issue 6534 (_sum, _count, then the buckets), that slot was overwritten by the first finite bucket, the "+Inf" sample was skipped, and the "+Inf" bucket was decoded as zero while _count kept its value. Store the value of the "+Inf" bucket sample in its own slot, and only use _count for the histogram count. Signed-off-by: Chris Burr --- src/cmt_decode_prometheus.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/cmt_decode_prometheus.c b/src/cmt_decode_prometheus.c index 7d462d44..39dc413e 100644 --- a/src/cmt_decode_prometheus.c +++ b/src/cmt_decode_prometheus.c @@ -543,23 +543,26 @@ static int add_metric_histogram(struct cmt_decode_prometheus_context *context) sample = cfl_list_entry(head, struct cmt_decode_prometheus_context_sample, _head); switch (sample->type) { case CMT_DECODE_PROMETHEUS_CONTEXT_SAMPLE_TYPE_BUCKET: - if (bucket_index == bucket_count) { - /* probably last bucket, which has "Inf" */ + if (bucket_index > bucket_count) { break; } - if (!sample->label_values[le_label_index] || - sample->label_values[le_label_index][0] == '\0') { - ret = report_error(context, - CMT_DECODE_PROMETHEUS_SYNTAX_ERROR, - "missing histogram bucket \"le\" value"); - goto end; - } - if (parse_double(sample->label_values[le_label_index], - buckets + bucket_index)) { - ret = report_error(context, - CMT_DECODE_PROMETHEUS_CMT_CREATE_ERROR, - "failed to parse bucket"); - goto end; + /* the last bucket is "+Inf": it has no boundary to parse and + * its value goes to the extra slot of bucket_defaults */ + if (bucket_index < bucket_count) { + if (!sample->label_values[le_label_index] || + sample->label_values[le_label_index][0] == '\0') { + ret = report_error(context, + CMT_DECODE_PROMETHEUS_SYNTAX_ERROR, + "missing histogram bucket \"le\" value"); + goto end; + } + if (parse_double(sample->label_values[le_label_index], + buckets + bucket_index)) { + ret = report_error(context, + CMT_DECODE_PROMETHEUS_CMT_CREATE_ERROR, + "failed to parse bucket"); + goto end; + } } if (parse_uint64(sample->value1, bucket_defaults + bucket_index)) { @@ -617,7 +620,6 @@ static int add_metric_histogram(struct cmt_decode_prometheus_context *context) count = (uint64_t)count_dbl; } } - bucket_defaults[bucket_index] = count; has_count = CMT_TRUE; if (!timestamp) { From 20269b53c2bac8261a69bf81ce5d16eae8408449 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 9 Nov 2026 13:50:47 +0100 Subject: [PATCH 5/5] tests: check histogram +Inf bucket when _count comes first Decode histograms whose _count is written before the buckets, with and without _sum, and check the "+Inf" bucket values. The expected output of the fluent-bit issue 6534 test had every "+Inf" bucket at zero, which was the decoding error itself: they now match the input and _count. Signed-off-by: Chris Burr --- tests/prometheus_parser.c | 64 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/tests/prometheus_parser.c b/tests/prometheus_parser.c index b9185481..0548dddb 100644 --- a/tests/prometheus_parser.c +++ b/tests/prometheus_parser.c @@ -1661,7 +1661,7 @@ void test_issue_fluent_bit_6534() "dotnet_threadpool_queue_length_bucket{le=\"10.0\"} 321733 0\n" "dotnet_threadpool_queue_length_bucket{le=\"100.0\"} 321733 0\n" "dotnet_threadpool_queue_length_bucket{le=\"1000.0\"} 321733 0\n" - "dotnet_threadpool_queue_length_bucket{le=\"+Inf\"} 0 0\n" + "dotnet_threadpool_queue_length_bucket{le=\"+Inf\"} 321733 0\n" "dotnet_threadpool_queue_length_sum 5 0\n" "dotnet_threadpool_queue_length_count 321733 0\n" "# HELP dotnet_gc_pause_seconds The amount of time execution was paused for garbage collection\n" @@ -1673,7 +1673,7 @@ void test_issue_fluent_bit_6534() "dotnet_gc_pause_seconds_bucket{le=\"0.5\"} 759 0\n" "dotnet_gc_pause_seconds_bucket{le=\"1.0\"} 759 0\n" "dotnet_gc_pause_seconds_bucket{le=\"10.0\"} 759 0\n" - "dotnet_gc_pause_seconds_bucket{le=\"+Inf\"} 0 0\n" + "dotnet_gc_pause_seconds_bucket{le=\"+Inf\"} 759 0\n" "dotnet_gc_pause_seconds_sum 1.3192573999999997 0\n" "dotnet_gc_pause_seconds_count 759 0\n" "# HELP dotnet_gc_collection_seconds The amount of time spent running garbage collections\n" @@ -1685,7 +1685,7 @@ void test_issue_fluent_bit_6534() "dotnet_gc_collection_seconds_bucket{le=\"0.5\",gc_generation=\"1\",gc_type=\"non_concurrent_gc\"} 133 0\n" "dotnet_gc_collection_seconds_bucket{le=\"1.0\",gc_generation=\"1\",gc_type=\"non_concurrent_gc\"} 133 0\n" "dotnet_gc_collection_seconds_bucket{le=\"10.0\",gc_generation=\"1\",gc_type=\"non_concurrent_gc\"} 133 0\n" - "dotnet_gc_collection_seconds_bucket{le=\"+Inf\",gc_generation=\"1\",gc_type=\"non_concurrent_gc\"} 0 0\n" + "dotnet_gc_collection_seconds_bucket{le=\"+Inf\",gc_generation=\"1\",gc_type=\"non_concurrent_gc\"} 133 0\n" "dotnet_gc_collection_seconds_sum{gc_generation=\"1\",gc_type=\"non_concurrent_gc\"} 0.20421500000000006 0\n" "dotnet_gc_collection_seconds_count{gc_generation=\"1\",gc_type=\"non_concurrent_gc\"} 133 0\n" "# HELP dotnet_gc_collection_seconds The amount of time spent running garbage collections\n" @@ -1709,7 +1709,7 @@ void test_issue_fluent_bit_6534() "dotnet_gc_collection_seconds_bucket{le=\"0.5\",gc_generation=\"2\",gc_type=\"non_concurrent_gc\"} 8 0\n" "dotnet_gc_collection_seconds_bucket{le=\"1.0\",gc_generation=\"2\",gc_type=\"non_concurrent_gc\"} 8 0\n" "dotnet_gc_collection_seconds_bucket{le=\"10.0\",gc_generation=\"2\",gc_type=\"non_concurrent_gc\"} 8 0\n" - "dotnet_gc_collection_seconds_bucket{le=\"+Inf\",gc_generation=\"2\",gc_type=\"non_concurrent_gc\"} 0 0\n" + "dotnet_gc_collection_seconds_bucket{le=\"+Inf\",gc_generation=\"2\",gc_type=\"non_concurrent_gc\"} 8 0\n" "dotnet_gc_collection_seconds_sum{gc_generation=\"2\",gc_type=\"non_concurrent_gc\"} 0.093447800000000011 0\n" "dotnet_gc_collection_seconds_count{gc_generation=\"2\",gc_type=\"non_concurrent_gc\"} 8 0\n" "dotnet_gc_collection_seconds_bucket{le=\"0.001\",gc_generation=\"0\",gc_type=\"non_concurrent_gc\"} 127 0\n" @@ -1719,7 +1719,7 @@ void test_issue_fluent_bit_6534() "dotnet_gc_collection_seconds_bucket{le=\"0.5\",gc_generation=\"0\",gc_type=\"non_concurrent_gc\"} 618 0\n" "dotnet_gc_collection_seconds_bucket{le=\"1.0\",gc_generation=\"0\",gc_type=\"non_concurrent_gc\"} 618 0\n" "dotnet_gc_collection_seconds_bucket{le=\"10.0\",gc_generation=\"0\",gc_type=\"non_concurrent_gc\"} 618 0\n" - "dotnet_gc_collection_seconds_bucket{le=\"+Inf\",gc_generation=\"0\",gc_type=\"non_concurrent_gc\"} 0 0\n" + "dotnet_gc_collection_seconds_bucket{le=\"+Inf\",gc_generation=\"0\",gc_type=\"non_concurrent_gc\"} 618 0\n" "dotnet_gc_collection_seconds_sum{gc_generation=\"0\",gc_type=\"non_concurrent_gc\"} 0.85545190000000104 0\n" "dotnet_gc_collection_seconds_count{gc_generation=\"0\",gc_type=\"non_concurrent_gc\"} 618 0\n" ; @@ -2110,6 +2110,59 @@ void test_histogram_instances_without_sum() TEST_MSG("%s", errbuf); } +/* _count before the buckets must not overwrite or lose the "+Inf" bucket */ +void test_histogram_count_before_buckets() +{ + int status; + char errbuf[256]; + struct cmt *cmt; + struct cmt_decode_prometheus_parse_opts opts; + cfl_sds_t result; + + memset(&opts, 0, sizeof(opts)); + errbuf[0] = '\0'; + opts.errbuf = errbuf; + opts.errbuf_size = sizeof(errbuf); + + cmt = NULL; + status = cmt_decode_prometheus_create(&cmt, + "# TYPE a histogram\n" + "a_sum 5\n" + "a_count 3\n" + "a_bucket{le=\"1\"} 1\n" + "a_bucket{le=\"2\"} 2\n" + "a_bucket{le=\"+Inf\"} 3\n" + "# TYPE b histogram\n" + "b_count{code=\"200\"} 2\n" + "b_bucket{code=\"200\",le=\"1\"} 1\n" + "b_bucket{code=\"200\",le=\"+Inf\"} 2\n" + "b_count{code=\"500\"} 4\n" + "b_bucket{code=\"500\",le=\"1\"} 3\n" + "b_bucket{code=\"500\",le=\"+Inf\"} 4\n", 0, &opts); + TEST_CHECK(status == 0); + TEST_MSG("%s", errbuf); + if (status == 0) { + result = cmt_encode_prometheus_create(cmt, CMT_FALSE); + TEST_CHECK(strstr(result, "a_bucket{le=\"1.0\"} 1\n") != NULL); + TEST_CHECK(strstr(result, "a_bucket{le=\"2.0\"} 2\n") != NULL); + TEST_CHECK(strstr(result, "a_bucket{le=\"+Inf\"} 3\n") != NULL); + TEST_CHECK(strstr(result, "a_count 3\n") != NULL); + TEST_CHECK(strstr(result, + "b_bucket{le=\"1.0\",code=\"200\"} 1\n") != NULL); + TEST_CHECK(strstr(result, + "b_bucket{le=\"+Inf\",code=\"200\"} 2\n") != NULL); + TEST_CHECK(strstr(result, "b_count{code=\"200\"} 2\n") != NULL); + TEST_CHECK(strstr(result, + "b_bucket{le=\"1.0\",code=\"500\"} 3\n") != NULL); + TEST_CHECK(strstr(result, + "b_bucket{le=\"+Inf\",code=\"500\"} 4\n") != NULL); + TEST_CHECK(strstr(result, "b_count{code=\"500\"} 4\n") != NULL); + TEST_MSG("%s", result); + cfl_sds_destroy(result); + cmt_decode_prometheus_destroy(cmt); + } +} + TEST_LIST = { {"header_help", test_header_help}, {"header_type", test_header_type}, @@ -2155,5 +2208,6 @@ TEST_LIST = { {"duplicate_label_and_help", test_duplicate_label_and_help}, {"summary_instances_without_sum", test_summary_instances_without_sum}, {"histogram_instances_without_sum", test_histogram_instances_without_sum}, + {"histogram_count_before_buckets", test_histogram_count_before_buckets}, { 0 } };