From 37a670553dd438e5c1286d13e01d03d7b20fea3c Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 22 Oct 2026 17:13:11 -0600 Subject: [PATCH 1/7] cat: clear stale OTLP point metadata when the source has none Replacing a sample with one that carries no OTLP metadata kept the destination's old flags, exemplars and attributes on the new value. Signed-off-by: Eduardo Silva --- src/cmt_cat.c | 2 ++ tests/cat.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/cmt_cat.c b/src/cmt_cat.c index 40e09012..dbcd3bf5 100644 --- a/src/cmt_cat.c +++ b/src/cmt_cat.c @@ -506,6 +506,8 @@ static int copy_point_metadata(struct cmt_metric *dst, struct cmt_metric *src) struct cfl_kvlist *copy; if (src->otlp_metadata == NULL) { + cfl_kvlist_destroy(dst->otlp_metadata); + dst->otlp_metadata = NULL; return 0; } diff --git a/tests/cat.c b/tests/cat.c index 753dbd35..f27671aa 100644 --- a/tests/cat.c +++ b/tests/cat.c @@ -1265,6 +1265,60 @@ static uint64_t sample_flags(struct cmt_metric *sample) return UINT64_MAX; } +void test_cat_clears_stale_otlp_point_metadata() +{ + int label_count; + char *keys[] = {"key"}; + char *values[] = {"v"}; + struct cmt *dst; + struct cmt *src; + struct cmt_gauge *dst_gauge; + struct cmt_gauge *src_gauge; + struct cmt_metric *dst_sample; + struct cmt_metric *src_sample; + + /* Exercise both the static and label-indexed sample replacement paths. */ + for (label_count = 0; label_count <= 1; label_count++) { + dst = cmt_create(); + src = cmt_create(); + TEST_ASSERT(dst != NULL && src != NULL); + + dst_gauge = cmt_gauge_create(dst, "", "", "otlp_gauge", "otlp gauge", + label_count, keys); + src_gauge = cmt_gauge_create(src, "", "", "otlp_gauge", "otlp gauge", + label_count, keys); + TEST_ASSERT(dst_gauge != NULL && src_gauge != NULL); + TEST_ASSERT(cmt_gauge_set(dst_gauge, 100, 1.0, label_count, values) == 0); + TEST_ASSERT(cmt_gauge_set(src_gauge, 200, 2.5, label_count, values) == 0); + + dst_sample = cmt_map_metric_get(&dst_gauge->opts, dst_gauge->map, + label_count, values, CMT_FALSE); + src_sample = cmt_map_metric_get(&src_gauge->opts, src_gauge->map, + label_count, values, CMT_FALSE); + TEST_ASSERT(dst_sample != NULL && src_sample != NULL); + dst_sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(dst_sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(dst_sample->otlp_metadata, + "flags", 1) == 0); + cmt_metric_set_start_timestamp(dst_sample, 50); + TEST_ASSERT(src_sample->otlp_metadata == NULL); + + TEST_ASSERT(cmt_cat(dst, src) == 0); + TEST_CHECK(dst_sample->otlp_metadata == NULL); + TEST_CHECK(cmt_metric_get_value(dst_sample) == 2.5); + TEST_CHECK(cmt_metric_get_timestamp(dst_sample) == 200); + TEST_CHECK(!cmt_metric_has_start_timestamp(dst_sample)); + TEST_CHECK(cfl_list_size(&dst_gauge->map->metrics) == label_count); + + /* Repeated copies must also tolerate an already empty destination. */ + TEST_ASSERT(cmt_cat(dst, src) == 0); + TEST_CHECK(dst_sample->otlp_metadata == NULL); + + cmt_destroy(src); + cmt_destroy(dst); + } +} + /* Samples carrying OTLP metadata must merge by labels, never duplicate. */ void test_cat_otlp_metadata_samples_merge_by_labels() { @@ -1410,6 +1464,7 @@ void test_cat_otlp_metadata_distinct_points_kept() } TEST_LIST = { + {"cat_clears_stale_otlp_point_metadata", test_cat_clears_stale_otlp_point_metadata}, {"cat_otlp_metadata_distinct_points_kept", test_cat_otlp_metadata_distinct_points_kept}, {"cat_otlp_metadata_samples_merge_by_labels", test_cat_otlp_metadata_samples_merge_by_labels}, {"cat_many_contexts", test_cat_many_contexts}, From 99a8bd125dbd65af1c27cba63e96416d8d1d9bc9 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 22 Oct 2026 17:13:11 -0600 Subject: [PATCH 2/7] opentelemetry: keep points whose attributes hold empty nested values An AnyValue with no value set inside an array or kvlist attribute was reported as an allocation failure, dropping the whole request. It is now treated as absent metadata and the point keeps its label-derived form. Signed-off-by: Eduardo Silva --- src/cmt_decode_opentelemetry.c | 41 +++++++++++--- tests/opentelemetry.c | 100 +++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 8 deletions(-) diff --git a/src/cmt_decode_opentelemetry.c b/src/cmt_decode_opentelemetry.c index 2bb3e460..04210a15 100644 --- a/src/cmt_decode_opentelemetry.c +++ b/src/cmt_decode_opentelemetry.c @@ -38,7 +38,11 @@ #include #include -static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source); +/* Distinguish absent optional values from allocation failures. */ +#define CLONE_UNSUPPORTED_VALUE -1 + +static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source, + int *status); static int clone_array(struct cfl_array *target, Opentelemetry__Proto__Common__V1__ArrayValue *source); @@ -63,14 +67,18 @@ static int clone_exemplars_to_kvlist(struct cfl_kvlist *target, Opentelemetry__Proto__Metrics__V1__Exemplar **exemplars, size_t exemplar_count); -static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source) +static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source, + int *status) { struct cfl_kvlist *new_child_kvlist; struct cfl_array *new_child_array; struct cfl_variant *result_instance = NULL; int result; - if (source == NULL) { + *status = CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + if (source == NULL || source->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET) { + *status = CLONE_UNSUPPORTED_VALUE; return NULL; } if (source->value_case == OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE) { @@ -105,6 +113,7 @@ static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyVa result = clone_kvlist(new_child_kvlist, source->kvlist_value); if (result) { cfl_variant_destroy(result_instance); + *status = result; return NULL; } @@ -126,6 +135,7 @@ static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyVa result = clone_array(new_child_array, source->array_value); if (result) { cfl_variant_destroy(result_instance); + *status = result; return NULL; } @@ -134,6 +144,13 @@ static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyVa result_instance = cfl_variant_create_from_bytes((char *) source->bytes_value.data, source->bytes_value.len, CFL_FALSE); } + else { + *status = CLONE_UNSUPPORTED_VALUE; + } + + if (result_instance != NULL) { + *status = CMT_DECODE_OPENTELEMETRY_SUCCESS; + } return result_instance; } @@ -167,9 +184,9 @@ static int clone_array_entry(struct cfl_array *target, return CMT_DECODE_OPENTELEMETRY_SUCCESS; } - new_child_instance = clone_variant(source); + new_child_instance = clone_variant(source, &result); if (new_child_instance == NULL) { - return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + return result; } result = cfl_array_append(target, new_child_instance); @@ -214,10 +231,10 @@ static int clone_kvlist_entry(struct cfl_kvlist *target, return CMT_DECODE_OPENTELEMETRY_SUCCESS; } - new_child_instance = clone_variant(source->value); + new_child_instance = clone_variant(source->value, &result); if (new_child_instance == NULL) { - return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + return result; } result = cfl_kvlist_insert(target, source->key, new_child_instance); @@ -378,6 +395,7 @@ static int clone_point_attributes(struct cfl_kvlist *metadata, { struct cfl_kvlist *copy; size_t index; + int result; copy = cfl_kvlist_create(); if (copy == NULL) { @@ -390,7 +408,14 @@ static int clone_point_attributes(struct cfl_kvlist *metadata, OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET)) { continue; } - if (attributes[index] == NULL || clone_kvlist_entry(copy, attributes[index]) != 0) { + result = clone_kvlist_entry(copy, attributes[index]); + if (result == CLONE_UNSUPPORTED_VALUE) { + /* A partial typed tree would change the attributes. Retain the + * label-derived representation when any nested value is absent. */ + cfl_kvlist_destroy(copy); + return 0; + } + if (attributes[index] == NULL || result != 0) { cfl_kvlist_destroy(copy); return -1; } diff --git a/tests/opentelemetry.c b/tests/opentelemetry.c index 0bf8221e..a3618a4a 100644 --- a/tests/opentelemetry.c +++ b/tests/opentelemetry.c @@ -3220,7 +3220,107 @@ static void test_opentelemetry_repeated_unlabeled_points_last_wins(void) cfl_sds_destroy(payload); } +/* Empty nested AnyValues are optional metadata, not allocation failures. */ +static void test_otlp_nested_empty_attributes_keep_point(void) +{ + int mode; + size_t offset; + size_t size; + int result; + cfl_sds_t payload; + cfl_sds_t wire; + struct cfl_list contexts; + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Common__V1__AnyValue empty; + Opentelemetry__Proto__Common__V1__AnyValue nested; + Opentelemetry__Proto__Common__V1__AnyValue *items[] = {&empty}; + Opentelemetry__Proto__Common__V1__ArrayValue array; + Opentelemetry__Proto__Common__V1__KeyValue entry; + Opentelemetry__Proto__Common__V1__KeyValue *entries[] = {&entry}; + Opentelemetry__Proto__Common__V1__KeyValueList kvlist; + Opentelemetry__Proto__Common__V1__KeyValue *attribute; + Opentelemetry__Proto__Common__V1__AnyValue *original; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + /* An array containing an empty value, a kvlist with no value, and a + * kvlist containing an explicitly empty value must all remain usable. */ + for (mode = 0; mode < 3; mode++) { + payload = generate_gauge_int_otlp_payload_with_many_attributes(2); + TEST_ASSERT(payload != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(payload), (const uint8_t *) payload); + TEST_ASSERT(request != NULL); + cfl_sds_destroy(payload); + attribute = request->resource_metrics[0]->scope_metrics[0]->metrics[0]->gauge->data_points[0]->attributes[0]; + original = attribute->value; + opentelemetry__proto__common__v1__any_value__init(&empty); + opentelemetry__proto__common__v1__any_value__init(&nested); + opentelemetry__proto__common__v1__array_value__init(&array); + opentelemetry__proto__common__v1__key_value__init(&entry); + opentelemetry__proto__common__v1__key_value_list__init(&kvlist); + array.n_values = 1; + array.values = items; + entry.key = "empty"; + entry.value = mode == 2 ? &empty : NULL; + kvlist.n_values = 1; + kvlist.values = entries; + if (mode == 0) { + nested.value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE; + nested.array_value = &array; + } + else { + nested.value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE; + nested.kvlist_value = &kvlist; + } + attribute->value = &nested; + size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(request); + payload = cfl_sds_create_size(size); + TEST_ASSERT(payload != NULL); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(request, (uint8_t *) payload); + cfl_sds_len_set(payload, size); + attribute->value = original; + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + + offset = 0; + result = cmt_decode_opentelemetry_create(&contexts, payload, size, &offset); + TEST_CHECK(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + ctx = cfl_list_entry_first(&contexts, struct cmt, _head); + TEST_ASSERT(ctx != NULL); + gauge = cfl_list_entry_first(&ctx->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_int64_value(sample) == 1); + TEST_CHECK(cmt_metric_get_timestamp(sample) == 123); + TEST_CHECK(cfl_kvlist_fetch(sample->otlp_metadata, "attributes") == NULL); + wire = cmt_encode_opentelemetry_create(ctx); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "g_many_attrs"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->as_int == 1); + TEST_ASSERT(metric->gauge->data_points[0]->n_attributes == 1); + TEST_CHECK(strcmp(metric->gauge->data_points[0]->attributes[0]->key, "attr_001") == 0); + TEST_CHECK(metric->gauge->data_points[0]->attributes[0]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE); + TEST_CHECK(strcmp(metric->gauge->data_points[0]->attributes[0]->value->string_value, + "value_001") == 0); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_decode_opentelemetry_destroy(&contexts); + } + cfl_sds_destroy(payload); + } +} + TEST_LIST = { + {"otlp_nested_empty_attributes_keep_point", test_otlp_nested_empty_attributes_keep_point}, {"otlp_unrepresentable_attribute_keeps_point", test_otlp_unrepresentable_attribute_keeps_point}, {"otlp_metadata_legacy_external_tree", test_otlp_metadata_legacy_external_tree}, {"otlp_metadata_deep_nesting", test_otlp_metadata_deep_nesting}, From b11af08e621759d391efc6c87badf484f69eee76 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 22 Oct 2026 17:13:11 -0600 Subject: [PATCH 3/7] mpack: report oversized array counts as zero when peeking Bucket, quantile and exponential bucket decoders allocate from the peeked count before cmt_mpack_unpack_array() rejects counts above CMT_MPACK_MAX_ARRAY_ENTRY_COUNT, so a corrupt chunk could request gigabytes before failing. Peeking now reports such counts as zero and the array read still fails. Signed-off-by: Eduardo Silva --- src/cmt_mpack_utils.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cmt_mpack_utils.c b/src/cmt_mpack_utils.c index ea67d5ef..ddbc4f9a 100644 --- a/src/cmt_mpack_utils.c +++ b/src/cmt_mpack_utils.c @@ -332,5 +332,11 @@ int cmt_mpack_peek_array_length(mpack_reader_t *reader) return 0; } + /* Callers size allocations from this value before the array is read, so + * a declared count that cmt_mpack_unpack_array() rejects reports zero. */ + if (CMT_MPACK_MAX_ARRAY_ENTRY_COUNT < mpack_tag_array_count(&tag)) { + return 0; + } + return mpack_tag_array_count(&tag); } From 46c100250741145c7876ae43643b85fcb906a256 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 22 Oct 2026 17:13:11 -0600 Subject: [PATCH 4/7] otlp_metadata: bind point metadata by point identity The v2.3.0 native metadata sidecar matched point metadata by position. Released readers relay a chunk with the sidecar unchanged but can add, remove, replace or reorder points, so flags, exemplars and typed attributes landed on other series. The writer now records an identity for every point under a new "point_ids" key, which v2.3.0 readers ignore: series hash, time, label values written as the codec writes labels, a fingerprint of the complete value as the codec carries it, and the start time when set. The series hash only accelerates the comparison, since it concatenates label values without boundaries. The reader looks every point up through the identifiers sorted once per family, so restoring takes n log n time, and an identity recorded more than once binds nothing. Sidecars without identifiers keep the positional rule. Legacy external entries, keyed by series hash and time only, are pruned in one pass. With identifiers, an entry stays only when every point recorded under its key was bound by a unique identity and has no native metadata that reads back. Without them, only entries of restored points are dropped. Whether the sidecar is restored, skipped, rejected or absent, an entry whose key several decoded samples share is dropped. Families whose metadata exists only in the legacy tree keep it. Signed-off-by: Eduardo Silva --- src/cmt_encode_opentelemetry.c | 20 +- src/cmt_otlp_metadata.c | 805 +++++++++++++++++++++++++++++++-- tests/msgpack_abi.c | 9 +- tests/opentelemetry.c | 697 ++++++++++++++++++++++++++++ 4 files changed, 1487 insertions(+), 44 deletions(-) diff --git a/src/cmt_encode_opentelemetry.c b/src/cmt_encode_opentelemetry.c index f384e13c..d3b8fcbc 100644 --- a/src/cmt_encode_opentelemetry.c +++ b/src/cmt_encode_opentelemetry.c @@ -145,7 +145,8 @@ static char *map_type_to_key(int map_type) } } -static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map) +static struct cfl_kvlist *get_legacy_metric_otlp_metadata_context(struct cmt *cmt, + struct cmt_map *map) { struct cfl_kvlist *otlp_root; struct cfl_kvlist *metrics_root; @@ -155,10 +156,6 @@ static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, stru return NULL; } - if (map->otlp_metadata) { - return map->otlp_metadata; - } - otlp_root = fetch_metadata_kvlist_key(cmt->external_metadata, "otlp"); if (otlp_root == NULL) { return NULL; @@ -177,6 +174,15 @@ static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, stru return fetch_metadata_kvlist_key(type_root, map->opts->fqname); } +static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map) +{ + if (map != NULL && map->otlp_metadata) { + return map->otlp_metadata; + } + + return get_legacy_metric_otlp_metadata_context(cmt, map); +} + static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map, struct cmt_metric *sample) @@ -189,7 +195,9 @@ static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, return sample->otlp_metadata; } - metric_context = get_metric_otlp_metadata_context(cmt, map); + /* Native family metadata never carries points. A point an intermediary + * left unbound keeps its legacy entry, as released readers do. */ + metric_context = get_legacy_metric_otlp_metadata_context(cmt, map); if (metric_context == NULL) { return NULL; } diff --git a/src/cmt_otlp_metadata.c b/src/cmt_otlp_metadata.c index d0b67041..72487a1c 100644 --- a/src/cmt_otlp_metadata.c +++ b/src/cmt_otlp_metadata.c @@ -26,7 +26,10 @@ #include #include #include +#include +#include +#include #include #include #include @@ -240,13 +243,152 @@ static int pack_optional_map(mpack_writer_t *writer, struct cfl_kvlist *map) return 0; } +static void fingerprint_u64(cfl_hash_state_t *state, uint64_t value) +{ + unsigned char bytes[8]; + int index; + + /* byte order independent: writer and reader may differ */ + for (index = 0; index < 8; index++) { + bytes[index] = (unsigned char) (value >> (index * 8)); + } + cfl_hash_64bits_update(state, bytes, sizeof(bytes)); +} + +static uint64_t double_bits(double value) +{ + uint64_t bits; + + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +/* + * Fingerprint of the complete value a point carries through the MessagePack + * codec, read as pack_metric() reads it, distributions included. Points of + * one series and time window that differ only in what the labels cannot + * hold, such as attribute types, usually differ in value too. + */ +static int point_fingerprint(struct cmt_map *map, struct cmt_metric *sample, + uint64_t *fingerprint) +{ + struct cmt_exp_histogram_snapshot snapshot; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + cfl_hash_state_t state; + int64_t int64_value; + uint64_t uint64_value; + size_t index; + int type; + + cfl_hash_64bits_reset(&state); + fingerprint_u64(&state, map->type); + if (map->type == CMT_HISTOGRAM) { + histogram = (struct cmt_histogram *) map->parent; + for (index = 0; index <= histogram->buckets->count; index++) { + fingerprint_u64(&state, cmt_metric_hist_get_value(sample, index)); + } + fingerprint_u64(&state, cmt_metric_hist_get_count_value(sample)); + fingerprint_u64(&state, double_bits(cmt_metric_hist_get_sum_value(sample))); + } + else if (map->type == CMT_SUMMARY) { + summary = (struct cmt_summary *) map->parent; + fingerprint_u64(&state, cmt_atomic_load(&sample->sum_quantiles_set)); + for (index = 0; sample->sum_quantiles != NULL && + index < summary->quantiles_count; index++) { + fingerprint_u64(&state, cmt_atomic_load(&sample->sum_quantiles[index])); + } + fingerprint_u64(&state, cmt_summary_get_count_value(sample)); + fingerprint_u64(&state, double_bits(cmt_summary_get_sum_value(sample))); + } + else if (map->type == CMT_EXP_HISTOGRAM) { + if (cmt_metric_exp_hist_get_snapshot(sample, &snapshot) != 0) { + return -1; + } + fingerprint_u64(&state, (uint64_t) (int64_t) snapshot.scale); + fingerprint_u64(&state, snapshot.zero_count); + fingerprint_u64(&state, double_bits(snapshot.zero_threshold)); + fingerprint_u64(&state, (uint64_t) (int64_t) snapshot.positive_offset); + fingerprint_u64(&state, snapshot.positive_count); + for (index = 0; index < snapshot.positive_count; index++) { + fingerprint_u64(&state, snapshot.positive_buckets[index]); + } + fingerprint_u64(&state, (uint64_t) (int64_t) snapshot.negative_offset); + fingerprint_u64(&state, snapshot.negative_count); + for (index = 0; index < snapshot.negative_count; index++) { + fingerprint_u64(&state, snapshot.negative_buckets[index]); + } + fingerprint_u64(&state, snapshot.count); + fingerprint_u64(&state, snapshot.sum_set); + fingerprint_u64(&state, snapshot.sum); + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + } + else { + cmt_metric_get_value_snapshot(sample, &type, &int64_value, &uint64_value); + fingerprint_u64(&state, type); + if (type == CMT_METRIC_VALUE_INT64) { + fingerprint_u64(&state, (uint64_t) int64_value); + } + else if (type == CMT_METRIC_VALUE_UINT64) { + fingerprint_u64(&state, uint64_value); + } + else { + fingerprint_u64(&state, double_bits(cmt_metric_get_value(sample))); + } + } + *fingerprint = cfl_hash_64bits_digest(&state); + return 0; +} + +/* + * A point is identified as in cmt_cat() by label values, time and start + * time, plus a fingerprint of its values. The series hash concatenates label + * values without boundaries, so distinct label sets can share it: it only + * accelerates the comparison. Values are written as pack_metric() writes + * labels, so they compare against the labels a reader decodes. The start + * time is appended only when set, so its absence stays distinct. A point + * whose fingerprint cannot be computed records zero and is unlikely to bind. + */ +static void pack_point_id(mpack_writer_t *writer, struct cmt_map *map, + struct cmt_metric *sample) +{ + int has_start; + uint64_t fingerprint; + struct cfl_list *head; + struct cmt_map_label *label; + + if (point_fingerprint(map, sample, &fingerprint) != 0) { + fingerprint = 0; + } + has_start = cmt_metric_has_start_timestamp(sample); + mpack_start_array(writer, has_start ? 5 : 4); + mpack_write_uint(writer, sample->hash); + mpack_write_uint(writer, cmt_metric_get_timestamp(sample)); + mpack_start_array(writer, cfl_list_size(&sample->labels)); + cfl_list_foreach(head, &sample->labels) { + label = cfl_list_entry(head, struct cmt_map_label, _head); + if (label->name == NULL) { + mpack_write_nil(writer); + } + else { + mpack_write_cstr(writer, label->name); + } + } + mpack_finish_array(writer); + mpack_write_uint(writer, fingerprint); + if (has_start) { + mpack_write_uint(writer, cmt_metric_get_start_timestamp(sample)); + } + mpack_finish_array(writer); +} + static int pack_map_metadata(struct cmt_map *map, void *data) { mpack_writer_t *writer = data; struct cfl_list *head; struct cmt_metric *sample; - mpack_start_map(writer, 5); + mpack_start_map(writer, 6); mpack_write_cstr(writer, "type"); mpack_write_int(writer, map->type); mpack_write_cstr(writer, "name"); @@ -270,6 +412,21 @@ static int pack_map_metadata(struct cmt_map *map, void *data) } } mpack_finish_array(writer); + + /* Identity of each entry in "points". An intermediary + * that edits points without updating the sidecar (any released reader) + * changes them, so readers bind by identity instead of by position. + * Readers of v2.3.0 ignore this key. */ + mpack_write_cstr(writer, "point_ids"); + mpack_start_array(writer, cfl_list_size(&map->metrics) + !!map->metric_static_set); + if (map->metric_static_set) { + pack_point_id(writer, map, &map->metric); + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + pack_point_id(writer, map, sample); + } + mpack_finish_array(writer); mpack_finish_map(writer); return 0; } @@ -533,11 +690,20 @@ struct staged_family_metadata { struct cmt_map *map; struct cfl_kvlist *family; struct cfl_kvlist **points; + struct cfl_array *point_ids; + size_t *order; + unsigned char *point_flags; size_t point_count; int allow_reset; int restore_points; }; +/* point_flags bits, per staged point: identity recorded more than once, + * native metadata read back, bound to a decoded sample */ +#define POINT_AMBIGUOUS 1 +#define POINT_HAS_METADATA 2 +#define POINT_MATCHED 4 + struct restore_state { struct cfl_array *families; size_t index; @@ -580,6 +746,268 @@ static int restore_optional_map(struct cfl_kvlist **target, struct cfl_variant * return 0; } +/* Unsigned values are read back as CFL_VARIANT_INT with the same bits. */ +static int is_integer(struct cfl_variant *value) +{ + return value->type == CFL_VARIANT_INT || value->type == CFL_VARIANT_UINT; +} + +static uint64_t integer_bits(struct cfl_variant *value) +{ + if (value->type == CFL_VARIANT_INT) { + return (uint64_t) value->data.as_int64; + } + return value->data.as_uint64; +} + +static int label_values_valid(struct cfl_variant *values) +{ + struct cfl_array *array; + size_t index; + + if (values->type != CFL_VARIANT_ARRAY) { + return CMT_FALSE; + } + array = values->data.as_array; + for (index = 0; index < array->entry_count; index++) { + if (array->entries[index]->type != CFL_VARIANT_STRING && + array->entries[index]->type != CFL_VARIANT_NULL) { + return CMT_FALSE; + } + } + return CMT_TRUE; +} + +/* Each identifier is [series hash, timestamp, label values, fingerprint] + * with the start timestamp appended when set, one per "points" entry. */ +static int point_ids_valid(struct cfl_variant *ids, size_t count) +{ + struct cfl_array *id; + size_t index; + + if (ids->type != CFL_VARIANT_ARRAY || ids->data.as_array->entry_count != count) { + return CMT_FALSE; + } + for (index = 0; index < count; index++) { + if (ids->data.as_array->entries[index]->type != CFL_VARIANT_ARRAY) { + return CMT_FALSE; + } + id = ids->data.as_array->entries[index]->data.as_array; + if (id->entry_count < 4 || id->entry_count > 5 || + !is_integer(id->entries[0]) || !is_integer(id->entries[1]) || + !label_values_valid(id->entries[2]) || !is_integer(id->entries[3]) || + (id->entry_count == 5 && !is_integer(id->entries[4]))) { + return CMT_FALSE; + } + } + return CMT_TRUE; +} + +static int compare_u64(uint64_t left, uint64_t right) +{ + return left < right ? -1 : left > right; +} + +/* Order a label value against one read back from an identifier: nil first, + * then by length and bytes, so values keep their boundaries. */ +static int compare_label_value(struct cfl_variant *value, cfl_sds_t label) +{ + size_t length; + int result; + + if (value->type == CFL_VARIANT_NULL || label == NULL) { + return (value->type != CFL_VARIANT_NULL) - (label != NULL); + } + length = cfl_sds_len(value->data.as_string); + if (length != cfl_sds_len(label)) { + return length < cfl_sds_len(label) ? -1 : 1; + } + result = memcmp(value->data.as_string, label, length); + return (result > 0) - (result < 0); +} + +static int compare_id_header(struct cfl_array *id, uint64_t hash, uint64_t timestamp, + uint64_t fingerprint, int has_start, uint64_t start, + size_t label_count) +{ + int result; + + result = compare_u64(integer_bits(id->entries[0]), hash); + if (result == 0) { + result = compare_u64(integer_bits(id->entries[1]), timestamp); + } + if (result == 0) { + result = compare_u64(integer_bits(id->entries[3]), fingerprint); + } + if (result == 0) { + result = (id->entry_count == 5) - (has_start != 0); + } + if (result == 0 && has_start) { + result = compare_u64(integer_bits(id->entries[4]), start); + } + if (result == 0) { + result = compare_u64(id->entries[2]->data.as_array->entry_count, label_count); + } + return result; +} + +/* Order an identifier against a decoded sample with its fingerprint. */ +static int compare_id_sample(struct cfl_array *id, struct cmt_metric *sample, + uint64_t fingerprint) +{ + struct cfl_array *values; + struct cfl_list *head; + struct cmt_map_label *label; + size_t index = 0; + int has_start; + int result; + + has_start = cmt_metric_has_start_timestamp(sample); + result = compare_id_header(id, sample->hash, cmt_metric_get_timestamp(sample), + fingerprint, has_start, + has_start ? cmt_metric_get_start_timestamp(sample) : 0, + cfl_list_size(&sample->labels)); + values = id->entries[2]->data.as_array; + cfl_list_foreach(head, &sample->labels) { + if (result != 0) { + break; + } + label = cfl_list_entry(head, struct cmt_map_label, _head); + result = compare_label_value(values->entries[index++], label->name); + } + return result; +} + +/* Order two identifiers. */ +static int compare_ids(struct cfl_array *left, struct cfl_array *right) +{ + struct cfl_array *left_values; + struct cfl_array *right_values; + struct cfl_variant *value; + size_t index; + int has_start; + int result; + + has_start = right->entry_count == 5; + right_values = right->entries[2]->data.as_array; + result = compare_id_header(left, integer_bits(right->entries[0]), + integer_bits(right->entries[1]), + integer_bits(right->entries[3]), has_start, + has_start ? integer_bits(right->entries[4]) : 0, + right_values->entry_count); + left_values = left->entries[2]->data.as_array; + for (index = 0; result == 0 && index < left_values->entry_count; index++) { + value = right_values->entries[index]; + result = compare_label_value(left_values->entries[index], + value->type == CFL_VARIANT_NULL ? + NULL : value->data.as_string); + } + return result; +} + +static int compare_staged(struct cfl_array *ids, size_t left, size_t right) +{ + return compare_ids(ids->entries[left]->data.as_array, + ids->entries[right]->data.as_array); +} + +static void sift_down(struct cfl_array *ids, size_t *order, size_t root, size_t count) +{ + size_t child; + size_t swap; + + while ((child = root * 2 + 1) < count) { + if (child + 1 < count && compare_staged(ids, order[child], order[child + 1]) < 0) { + child++; + } + if (compare_staged(ids, order[root], order[child]) >= 0) { + return; + } + swap = order[root]; + order[root] = order[child]; + order[child] = swap; + root = child; + } +} + +/* Sort identifier positions in place. The comparison needs the identifiers, + * which qsort() cannot receive without shared state. */ +static void sort_point_ids(struct cfl_array *ids, size_t *order, size_t count) +{ + size_t index; + size_t swap; + + for (index = count / 2; index-- > 0;) { + sift_down(ids, order, index, count); + } + for (index = count; index-- > 1;) { + swap = order[0]; + order[0] = order[index]; + order[index] = swap; + sift_down(ids, order, 0, index); + } +} + +/* Find the staged entry recorded for this sample, or point_count. */ +static size_t find_point_id(struct staged_family_metadata *staged, struct cmt_metric *sample, + uint64_t fingerprint) +{ + size_t low = 0; + size_t high = staged->point_count; + size_t middle; + + while (low < high) { + middle = low + (high - low) / 2; + if (compare_id_sample(staged->point_ids->entries[staged->order[middle]] + ->data.as_array, sample, fingerprint) < 0) { + low = middle + 1; + } + else { + high = middle; + } + } + if (low >= staged->point_count || + compare_id_sample(staged->point_ids->entries[staged->order[low]]->data.as_array, + sample, fingerprint) != 0) { + return staged->point_count; + } + return staged->order[low]; +} + +/* + * Take the staged metadata recorded for this sample. Without identifiers + * (v2.3.0) points bind by position. Otherwise each sample is looked up by + * identity through the sorted identifiers, wherever it now sits. A sample + * whose identity is not recorded gets no metadata, and neither does one + * whose identity was recorded more than once: nothing tells those points + * apart, so whichever binding was chosen could be another point's. + */ +static struct cfl_kvlist *take_point_metadata(struct staged_family_metadata *staged, + struct cmt_metric *sample, size_t position) +{ + struct cfl_kvlist *metadata; + uint64_t fingerprint; + size_t index; + + if (staged->point_ids == NULL) { + index = position; + } + else { + if (point_fingerprint(staged->map, sample, &fingerprint) != 0) { + return NULL; + } + index = find_point_id(staged, sample, fingerprint); + if (index >= staged->point_count || + (staged->point_flags[index] & POINT_AMBIGUOUS)) { + return NULL; + } + staged->point_flags[index] |= POINT_MATCHED; + } + metadata = staged->points[index]; + staged->points[index] = NULL; + return metadata; +} + static int stage_map_metadata(struct cmt_map *map, void *data) { struct restore_state *state = data; @@ -587,6 +1015,7 @@ static int stage_map_metadata(struct cmt_map *map, void *data) struct cfl_variant *family; struct cfl_variant *points; struct cfl_variant *reset; + struct cfl_variant *ids; size_t index; staged = &state->staged[state->index]; @@ -605,11 +1034,21 @@ static int stage_map_metadata(struct cmt_map *map, void *data) staged->family = NULL; } - /* Label-less entries fold into the static metric on decode, so the point - * list of that family may no longer line up: keep the family metadata - * but skip its point metadata rather than misattributing it. */ - staged->restore_points = points->data.as_array->entry_count == - cfl_list_size(&map->metrics) + !!map->metric_static_set; + ids = cfl_kvlist_fetch(family->data.as_kvlist, "point_ids"); + if (ids != NULL) { + /* Points are bound by labels, time and start time. Identifiers that + * cannot be read back leave the family without point metadata. */ + staged->restore_points = point_ids_valid(ids, points->data.as_array->entry_count); + staged->point_ids = staged->restore_points ? ids->data.as_array : NULL; + } + else { + /* v2.3.0 sidecars carry no identifiers and bind by position. Label-less + * entries fold into the static metric on decode, so the point list of + * that family may no longer line up: keep the family metadata but skip + * its point metadata rather than misattributing it. */ + staged->restore_points = points->data.as_array->entry_count == + cfl_list_size(&map->metrics) + !!map->metric_static_set; + } if (!staged->restore_points) { return 0; } @@ -621,23 +1060,287 @@ static int stage_map_metadata(struct cmt_map *map, void *data) return -1; } } + if (staged->point_ids != NULL && staged->point_count > 0) { + staged->point_flags = calloc(staged->point_count, sizeof(*staged->point_flags)); + if (!staged->point_flags) { + return -1; + } + } for (index = 0; index < staged->point_count; index++) { - /* a point whose metadata cannot be read back keeps none */ + /* a point whose metadata cannot be read back keeps none here, and + * keeps its legacy entry: only metadata read back supersedes it */ if (restore_optional_map(&staged->points[index], points->data.as_array->entries[index]) != 0) { staged->points[index] = NULL; } + if (staged->point_flags != NULL && staged->points[index] != NULL) { + staged->point_flags[index] |= POINT_HAS_METADATA; + } + } + if (staged->point_ids != NULL && staged->point_count > 0) { + staged->order = malloc(staged->point_count * sizeof(*staged->order)); + if (!staged->order) { + return -1; + } + for (index = 0; index < staged->point_count; index++) { + staged->order[index] = index; + } + sort_point_ids(staged->point_ids, staged->order, staged->point_count); + + for (index = 1; index < staged->point_count; index++) { + if (compare_staged(staged->point_ids, staged->order[index - 1], + staged->order[index]) == 0) { + staged->point_flags[staged->order[index - 1]] |= POINT_AMBIGUOUS; + staged->point_flags[staged->order[index]] |= POINT_AMBIGUOUS; + } + } + } + return 0; +} + +static struct cfl_kvlist *legacy_type_root(struct cmt *context, struct cmt_map *map) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(context->external_metadata, "otlp"); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + value = cfl_kvlist_fetch(value->data.as_kvlist, "metrics"); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + value = cfl_kvlist_fetch(value->data.as_kvlist, (char *) legacy_type_key(map->type)); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + return value->data.as_kvlist; +} + +static struct cfl_kvlist *legacy_family(struct cmt *context, struct cmt_map *map) +{ + struct cfl_kvlist *type_root; + struct cfl_variant *value; + + type_root = legacy_type_root(context, map); + if (type_root == NULL) { + return NULL; + } + value = cfl_kvlist_fetch(type_root, map->opts->fqname); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + return value->data.as_kvlist; +} + +/* Remove legacy containers left empty once native metadata replaced them. */ +static void prune_legacy_family(struct cmt *context, struct cmt_map *map) +{ + struct cfl_kvlist *family; + struct cfl_variant *datapoints; + + family = legacy_family(context, map); + if (family == NULL) { + return; + } + datapoints = cfl_kvlist_fetch(family, "datapoints"); + if (datapoints != NULL && datapoints->type == CFL_VARIANT_KVLIST && + cfl_kvlist_count(datapoints->data.as_kvlist) == 0) { + cfl_kvlist_remove(family, "datapoints"); + } + if (cfl_kvlist_count(family) == 0) { + cfl_kvlist_remove(legacy_type_root(context, map), map->opts->fqname); + } +} + +struct legacy_point_key { + uint64_t hash; + uint64_t timestamp; +}; + +static int compare_legacy_point_keys(const void *left, const void *right) +{ + const struct legacy_point_key *a = left; + const struct legacy_point_key *b = right; + + if (a->hash != b->hash) { + return a->hash < b->hash ? -1 : 1; + } + return (a->timestamp > b->timestamp) - (a->timestamp < b->timestamp); +} + +static void add_sample_key(struct legacy_point_key *keys, size_t *count, + struct cmt_metric *sample) +{ + keys[*count].hash = sample->hash; + keys[*count].timestamp = cmt_metric_get_timestamp(sample); + (*count)++; +} + +/* + * Append the keys two or more decoded samples share. Distinct series can share + * a series hash, and windows of one series a time, so such a legacy entry may + * be any of their points': it is no sample's fallback. Returns -1 when the + * samples cannot be gathered. + */ +static int add_shared_sample_keys(struct legacy_point_key *keys, size_t *count, + struct cmt_map *map) +{ + struct legacy_point_key *samples; + struct cfl_list *head; + size_t total = 0; + size_t index; + + samples = malloc((cfl_list_size(&map->metrics) + 1) * sizeof(*samples)); + if (samples == NULL) { + return -1; + } + if (map->metric_static_set) { + add_sample_key(samples, &total, &map->metric); + } + cfl_list_foreach(head, &map->metrics) { + add_sample_key(samples, &total, cfl_list_entry(head, struct cmt_metric, _head)); + } + qsort(samples, total, sizeof(*samples), compare_legacy_point_keys); + for (index = 1; index < total; index++) { + if (compare_legacy_point_keys(&samples[index - 1], &samples[index]) == 0 && + (*count == 0 || + compare_legacy_point_keys(&keys[*count - 1], &samples[index]) != 0)) { + keys[(*count)++] = samples[index]; + } } + free(samples); return 0; } -static void commit_map_metadata(struct staged_family_metadata *staged) +static void add_restored_key(struct legacy_point_key *keys, size_t *count, + struct cmt_metric *sample) +{ + if (sample->otlp_metadata != NULL) { + keys[*count].hash = sample->hash; + keys[*count].timestamp = cmt_metric_get_timestamp(sample); + (*count)++; + } +} + +/* A legacy "hash:timestamp" key names a restored point when it reads back + * as one of the restored keys in its canonical form, ignoring case as + * cfl_kvlist_remove() did. */ +static int legacy_key_listed(cfl_sds_t name, struct legacy_point_key *keys, size_t count) +{ + struct legacy_point_key key; + char canonical[64]; + char *end; + size_t length; + size_t index; + + key.hash = strtoull(name, &end, 16); + if (*end != ':') { + return CMT_FALSE; + } + key.timestamp = strtoull(end + 1, NULL, 10); + snprintf(canonical, sizeof(canonical), "%" PRIx64 ":%" PRIu64, + key.hash, key.timestamp); + length = strlen(canonical); + if (cfl_sds_len(name) != length) { + return CMT_FALSE; + } + for (index = 0; index < length; index++) { + if (tolower((unsigned char) name[index]) != canonical[index]) { + return CMT_FALSE; + } + } + return bsearch(&key, keys, count, sizeof(*keys), compare_legacy_point_keys) != NULL; +} + +/* + * Drop legacy copies the native metadata supersedes, in one pass over the + * legacy entries: removing them one by one would rescan the list for every + * point. + * + * With identifiers, a legacy entry, keyed by series hash and time only, stays + * only when every point recorded under its key was bound to a decoded sample + * by a unique identity and has no native metadata to read back: the entry is + * then that sample's own, as in released readers. It is dropped when native + * metadata replaced it, and when a recorded point was not bound (removed, + * changed by an intermediary, or ambiguous): the entry describes another or + * an older point. Legacy entries of points never recorded stay. + * Without identifiers (v2.3.0), only the copies of restored points are + * dropped, so points an intermediary left unbound keep what released readers + * keep. + */ +static void drop_legacy_points(struct cfl_kvlist *family, + struct staged_family_metadata *staged) { struct cmt_map *map = staged->map; + struct cfl_array *id; + size_t index; + struct legacy_point_key *keys; + struct cfl_variant *datapoints; + struct cfl_kvpair *pair; + struct cfl_list *head; + struct cfl_list *tmp; + struct cmt_metric *sample; + size_t count = 0; + + if (family == NULL) { + return; + } + datapoints = cfl_kvlist_fetch(family, "datapoints"); + if (datapoints == NULL || datapoints->type != CFL_VARIANT_KVLIST) { + return; + } + keys = malloc(((staged->point_ids != NULL ? staged->point_count : 0) + + cfl_list_size(&map->metrics) + 1) * sizeof(*keys)); + if (keys == NULL) { + /* no entry can be shown to be a safe fallback */ + cfl_kvlist_remove(family, "datapoints"); + return; + } + if (staged->point_ids != NULL) { + for (index = 0; index < staged->point_count; index++) { + if ((staged->point_flags[index] & POINT_MATCHED) && + !(staged->point_flags[index] & POINT_HAS_METADATA)) { + continue; + } + id = staged->point_ids->entries[index]->data.as_array; + keys[count].hash = integer_bits(id->entries[0]); + keys[count].timestamp = integer_bits(id->entries[1]); + count++; + } + } + else { + if (map->metric_static_set) { + add_restored_key(keys, &count, &map->metric); + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + add_restored_key(keys, &count, sample); + } + } + qsort(keys, count, sizeof(*keys), compare_legacy_point_keys); + cfl_list_foreach_safe(head, tmp, &datapoints->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (count > 0 && legacy_key_listed(pair->key, keys, count)) { + cfl_kvpair_destroy(pair); + } + } + free(keys); +} + +static void commit_map_metadata(struct cmt *context, + struct staged_family_metadata *staged) +{ + struct cmt_map *map = staged->map; + struct cfl_kvlist *legacy; struct cfl_list *head; struct cmt_metric *sample; size_t index = 0; + legacy = legacy_family(context, map); + if (legacy != NULL && staged->family != NULL) { + cfl_kvlist_remove(legacy, "metadata"); + } cfl_kvlist_destroy(map->otlp_metadata); map->otlp_metadata = staged->family; staged->family = NULL; @@ -645,19 +1348,20 @@ static void commit_map_metadata(struct staged_family_metadata *staged) ((struct cmt_counter *) map->parent)->allow_reset = staged->allow_reset; } if (!staged->restore_points) { + prune_legacy_family(context, map); return; } if (map->metric_static_set) { cfl_kvlist_destroy(map->metric.otlp_metadata); - map->metric.otlp_metadata = staged->points[index]; - staged->points[index++] = NULL; + map->metric.otlp_metadata = take_point_metadata(staged, &map->metric, index++); } cfl_list_foreach(head, &map->metrics) { sample = cfl_list_entry(head, struct cmt_metric, _head); cfl_kvlist_destroy(sample->otlp_metadata); - sample->otlp_metadata = staged->points[index]; - staged->points[index++] = NULL; + sample->otlp_metadata = take_point_metadata(staged, sample, index++); } + drop_legacy_points(legacy, staged); + prune_legacy_family(context, map); } static int check_map_metadata_shape(struct cmt_map *map, void *data) @@ -694,28 +1398,61 @@ static int check_map_metadata_shape(struct cmt_map *map, void *data) return 0; } -static void drop_legacy_family(struct cmt *context, struct cmt_map *map) +/* + * Drop legacy entries whose key two or more decoded samples share. Distinct + * series can share a series hash, and windows of one series a time, so such + * an entry may be any of their points': it is no sample's fallback. This + * holds whether native metadata was restored, skipped or absent, so it runs + * for every family. Entries one sample owns stay. + */ +static int drop_shared_legacy_keys(struct cmt_map *map, void *data) { - struct cfl_variant *otlp; - struct cfl_variant *metrics; - struct cfl_variant *type_root; + struct cmt *context = data; + struct legacy_point_key *keys; + struct cfl_kvlist *family; + struct cfl_variant *datapoints; + struct cfl_kvpair *pair; + struct cfl_list *head; + struct cfl_list *tmp; + size_t count = 0; - otlp = cfl_kvlist_fetch(context->external_metadata, "otlp"); - if (otlp == NULL || otlp->type != CFL_VARIANT_KVLIST) { - return; + family = legacy_family(context, map); + if (family == NULL) { + return 0; } - metrics = cfl_kvlist_fetch(otlp->data.as_kvlist, "metrics"); - if (metrics == NULL || metrics->type != CFL_VARIANT_KVLIST) { - return; + datapoints = cfl_kvlist_fetch(family, "datapoints"); + if (datapoints == NULL || datapoints->type != CFL_VARIANT_KVLIST) { + return 0; } - type_root = cfl_kvlist_fetch(metrics->data.as_kvlist, (char *) legacy_type_key(map->type)); - if (type_root == NULL || type_root->type != CFL_VARIANT_KVLIST) { - return; + keys = malloc((cfl_list_size(&map->metrics) + 1) * sizeof(*keys)); + if (keys == NULL || add_shared_sample_keys(keys, &count, map) != 0) { + /* no entry can be shown to be a safe fallback */ + cfl_kvlist_remove(family, "datapoints"); } - cfl_kvlist_remove(type_root->data.as_kvlist, map->opts->fqname); + else if (count > 0) { + cfl_list_foreach_safe(head, tmp, &datapoints->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (legacy_key_listed(pair->key, keys, count)) { + cfl_kvpair_destroy(pair); + } + } + } + free(keys); + prune_legacy_family(context, map); + return 0; } +static void restore_native_metadata(struct cmt *context); + int cmt_otlp_metadata_restore(struct cmt *context) +{ + restore_native_metadata(context); + visit_maps(context, drop_shared_legacy_keys, context); + /* Malformed optional metadata never invalidates the native metric payload. */ + return 0; +} + +static void restore_native_metadata(struct cmt *context) { struct cfl_variant *value; struct restore_state state; @@ -725,11 +1462,11 @@ int cmt_otlp_metadata_restore(struct cmt *context) value = cfl_kvlist_fetch(context->internal_metadata, METADATA_KEY); if (value == NULL) { - return 0; + return; } if (value->type != CFL_VARIANT_ARRAY) { cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); - return 0; + return; } state.families = value->data.as_array; state.index = 0; @@ -737,20 +1474,18 @@ int cmt_otlp_metadata_restore(struct cmt *context) if (result != 0 || state.index != state.families->entry_count) { /* Optional metadata must not invalidate an otherwise usable chunk. */ cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); - return 0; + return; } state.staged = calloc(state.families->entry_count, sizeof(*state.staged)); if (!state.staged && state.families->entry_count > 0) { cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); - return 0; + return; } state.index = 0; result = visit_maps(context, stage_map_metadata, &state); if (result == 0) { for (family_index = 0; family_index < state.families->entry_count; family_index++) { - /* the legacy copy written for older readers is superseded */ - drop_legacy_family(context, state.staged[family_index].map); - commit_map_metadata(&state.staged[family_index]); + commit_map_metadata(context, &state.staged[family_index]); } } for (family_index = 0; family_index < state.families->entry_count; family_index++) { @@ -761,11 +1496,11 @@ int cmt_otlp_metadata_restore(struct cmt *context) } } free(state.staged[family_index].points); + free(state.staged[family_index].order); + free(state.staged[family_index].point_flags); } free(state.staged); cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); - /* Malformed optional metadata never invalidates the native metric payload. */ - return 0; } int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample) diff --git a/tests/msgpack_abi.c b/tests/msgpack_abi.c index 2be211eb..44a903dd 100644 --- a/tests/msgpack_abi.c +++ b/tests/msgpack_abi.c @@ -420,8 +420,8 @@ static void test_point_metadata_and_unlabeled_windows(void) /* A further label-less point is the same series: like released readers, * it replaces the static metric (last one wins) instead of creating a - * duplicate series. Its point metadata no longer lines up with the - * sidecar and is skipped, while the values and the chunk survive. */ + * duplicate series. The sidecar binds point metadata by point identity, + * so the surviving point keeps its own metadata. */ source = create_static_gauge_with_metadata(); TEST_ASSERT(source != NULL); gauge = cfl_list_entry_first(&source->gauges, struct cmt_gauge, _head); @@ -443,7 +443,10 @@ static void test_point_metadata_and_unlabeled_windows(void) TEST_CHECK(cfl_list_size(&result->map->metrics) == 0); TEST_CHECK(cmt_metric_get_timestamp(&result->map->metric) == 2); TEST_CHECK(cmt_metric_get_value(&result->map->metric) == 2.0); - TEST_CHECK(result->map->metric.otlp_metadata == NULL); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "flags"); + TEST_ASSERT(value != NULL); + TEST_CHECK((value->type == CFL_VARIANT_UINT && value->data.as_uint64 == 3) || + (value->type == CFL_VARIANT_INT && value->data.as_int64 == 3)); cmt_decode_msgpack_destroy(decoded); cmt_encode_msgpack_destroy(buffer); cmt_destroy(source); diff --git a/tests/opentelemetry.c b/tests/opentelemetry.c index a3618a4a..074b8f52 100644 --- a/tests/opentelemetry.c +++ b/tests/opentelemetry.c @@ -3319,8 +3319,705 @@ static void test_otlp_nested_empty_attributes_keep_point(void) } } +static void test_otlp_restore_preserves_legacy_only_family(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *legacy; + struct cmt_gauge *native; + struct cfl_kvlist *external = NULL; + struct cfl_variant *node; + mpack_writer_t writer; + mpack_reader_t reader; + char *buffer = NULL; + size_t size = 0; + size_t offset = 0; + cfl_sds_t wire; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + legacy = cmt_gauge_create(ctx, "", "", "legacy_only", "legacy", 0, NULL); + native = cmt_gauge_create(ctx, "", "", "native", "native", 0, NULL); + TEST_ASSERT(legacy != NULL && native != NULL); + TEST_ASSERT(cmt_gauge_set(legacy, 100, 7, 0, NULL) == 0); + TEST_ASSERT(cmt_gauge_set(native, 200, 9, 0, NULL) == 0); + legacy->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(legacy->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(legacy->map->metric.otlp_metadata, "flags", 1) == 0); + + /* Build an external legacy tree, then leave this family with no native + * metadata. The other family forces a native sidecar for both maps. */ + mpack_writer_init_growable(&writer, &buffer, &size); + TEST_ASSERT(cmt_otlp_metadata_pack_external(&writer, ctx) == 0); + TEST_ASSERT(mpack_writer_destroy(&writer) == mpack_ok); + mpack_reader_init_data(&reader, buffer, size); + TEST_ASSERT(unpack_cfl_kvlist(&reader, &external) == 0); + TEST_ASSERT(mpack_reader_destroy(&reader) == mpack_ok); + free(buffer); + cfl_kvlist_destroy(ctx->external_metadata); + ctx->external_metadata = external; + cfl_kvlist_destroy(legacy->map->metric.otlp_metadata); + legacy->map->metric.otlp_metadata = NULL; + native->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(native->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(native->map->metric.otlp_metadata, "flags", 2) == 0); + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + node = cfl_kvlist_fetch(decoded->external_metadata, "otlp"); + TEST_ASSERT(node != NULL); + node = cfl_kvlist_fetch(node->data.as_kvlist, "metrics"); + TEST_ASSERT(node != NULL); + node = cfl_kvlist_fetch(node->data.as_kvlist, "gauge"); + TEST_ASSERT(node != NULL); + TEST_CHECK(cfl_kvlist_fetch(node->data.as_kvlist, "legacy_only") != NULL); + TEST_CHECK(cfl_kvlist_fetch(node->data.as_kvlist, "native") == NULL); + + wire = cmt_encode_opentelemetry_create(decoded); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "legacy_only"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->flags == 1); + metric = find_otlp_metric_by_name(request, "native"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->flags == 2); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* + * Released readers decode the sidecar and the legacy tree as opaque context + * metadata and write them back unchanged, even after editing points. These + * helpers capture both as such a relay would see them. + */ +static struct cfl_kvlist *relay_snapshot(struct cmt *ctx, int internal) +{ + struct cfl_kvlist *copy = NULL; + mpack_writer_t writer; + mpack_reader_t reader; + char *buffer = NULL; + size_t size = 0; + int result; + + mpack_writer_init_growable(&writer, &buffer, &size); + if (internal) { + result = cmt_otlp_metadata_pack(&writer, ctx); + } + else { + result = cmt_otlp_metadata_pack_external(&writer, ctx); + } + if (mpack_writer_destroy(&writer) != mpack_ok || result != 0) { + free(buffer); + return NULL; + } + mpack_reader_init_data(&reader, buffer, size); + result = unpack_cfl_kvlist(&reader, ©); + if (mpack_reader_destroy(&reader) != mpack_ok || result != 0) { + cfl_kvlist_destroy(copy); + copy = NULL; + } + free(buffer); + return copy; +} + +/* Replace native metadata with the captured opaque copies, as a released + * relay holds nothing else. */ +static void relay_install(struct cmt *ctx, struct cmt_map *map, + struct cfl_kvlist *internal, struct cfl_kvlist *external) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + cfl_kvlist_destroy(map->otlp_metadata); + map->otlp_metadata = NULL; + cfl_kvlist_destroy(map->metric.otlp_metadata); + map->metric.otlp_metadata = NULL; + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = NULL; + } + cfl_kvlist_destroy(ctx->internal_metadata); + ctx->internal_metadata = internal; + cfl_kvlist_destroy(ctx->external_metadata); + ctx->external_metadata = external; +} + +/* Remove the point identifiers, producing a v2.3.0 sidecar. */ +static void relay_strip_point_ids(struct cfl_kvlist *internal) +{ + struct cfl_variant *families; + size_t index; + + families = cfl_kvlist_fetch(internal, "__cmetrics_native_metadata_v1"); + TEST_ASSERT(families != NULL && families->type == CFL_VARIANT_ARRAY); + for (index = 0; index < families->data.as_array->entry_count; index++) { + cfl_kvlist_remove(families->data.as_array->entries[index]->data.as_kvlist, + "point_ids"); + } +} + +static struct cmt_gauge *relay_gauge(struct cmt *ctx, uint64_t flags_a, uint64_t flags_b, + int family_metadata) +{ + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *metadata; + char *values[2][1] = {{"a"}, {"b"}}; + uint64_t flags[2]; + int index; + + flags[0] = flags_a; + flags[1] = flags_b; + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 1, (char *[]) {"site"}); + TEST_ASSERT(gauge != NULL); + for (index = 0; index < 2; index++) { + TEST_ASSERT(cmt_gauge_set(gauge, 100, index + 1, 1, values[index]) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, values[index], CMT_FALSE); + TEST_ASSERT(sample != NULL); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", flags[index]) == 0); + } + if (family_metadata) { + gauge->map->otlp_metadata = cfl_kvlist_create(); + metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(metadata, "owner", "relay") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", metadata) == 0); + } + return gauge; +} + +static struct cmt *relay_round_trip(struct cmt *ctx, char **buffer) +{ + struct cmt *decoded = NULL; + size_t size; + size_t offset = 0; + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, *buffer, size, &offset) == 0); + TEST_CHECK(offset == size); + return decoded; +} + +static int relay_otlp_flags(struct cmt *ctx, char *site, uint32_t *flags) +{ + cfl_sds_t wire; + size_t index; + int found = CMT_FALSE; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *point; + + wire = cmt_encode_opentelemetry_create(ctx); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "relay"); + TEST_ASSERT(metric != NULL && metric->gauge != NULL); + for (index = 0; index < metric->gauge->n_data_points; index++) { + point = metric->gauge->data_points[index]; + if (point->n_attributes == 1 && + strcmp(point->attributes[0]->value->string_value, site) == 0) { + *flags = point->flags; + found = CMT_TRUE; + } + } + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + return found; +} + +/* A released relay replaces series "a" with "c" without updating the sidecar. + * Metadata recorded for "a" must not be attached to another series. */ +static void test_otlp_restore_binds_points_by_identity(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 1, 0, CMT_FALSE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + cmt_map_metric_destroy(cmt_map_metric_get(&gauge->opts, gauge->map, 1, + (char *[]) {"a"}, CMT_FALSE)); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 3, 1, (char *[]) {"c"}) == 0); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 0); + TEST_CHECK(relay_otlp_flags(decoded, "c", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* A released relay records a new observation for "b" without updating the + * sidecar. Metadata of the previous observation must not be restored. */ +static void test_otlp_restore_drops_stale_observation(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 0, 1, CMT_FALSE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + TEST_ASSERT(cmt_gauge_set(gauge, 200, 9, 1, (char *[]) {"b"}) == 0); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 0); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* A relay that reverses a large family must not make restoration scan the + * identifiers once per point, and every point keeps its own metadata. */ +static void test_otlp_restore_binds_reordered_points(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + struct cfl_variant *flags; + struct cfl_list reversed; + struct cfl_list *head; + struct cfl_list *tmp; + struct cmt_map_label *label; + char value[16]; + char *values[1] = {value}; + char *buffer; + int index; + int bound = 0; + const int count = 2000; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 1, (char *[]) {"site"}); + TEST_ASSERT(gauge != NULL); + for (index = 0; index < count; index++) { + snprintf(value, sizeof(value), "%d", index); + TEST_ASSERT(cmt_gauge_set(gauge, 100, index, 1, values) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, values, CMT_FALSE); + TEST_ASSERT(sample != NULL); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", index) == 0); + } + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_install(ctx, gauge->map, internal, external); + + cfl_list_init(&reversed); + cfl_list_foreach_safe(head, tmp, &gauge->map->metrics) { + cfl_list_del(head); + cfl_list_prepend(head, &reversed); + } + cfl_list_foreach_safe(head, tmp, &reversed) { + cfl_list_del(head); + cfl_list_add(head, &gauge->map->metrics); + } + + decoded = relay_round_trip(ctx, &buffer); + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + cfl_list_foreach(head, &gauge->map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + label = cfl_list_entry_first(&sample->labels, struct cmt_map_label, _head); + flags = cfl_kvlist_fetch(sample->otlp_metadata, "flags"); + if (flags != NULL && + (uint64_t) (flags->type == CFL_VARIANT_UINT ? flags->data.as_uint64 : + (uint64_t) flags->data.as_int64) == strtoull(label->name, NULL, 10)) { + bound++; + } + } + TEST_CHECK(bound == count); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* The series hash concatenates label values without boundaries, so these + * two series share it. A released relay drops the first: its metadata must + * not move to the survivor. */ +static void test_otlp_restore_binds_points_by_label_values(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_metric *first; + struct cmt_metric *second; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + struct cfl_variant *flags; + char *buffer; + char *a[] = {"ab", "c"}; + char *b[] = {"a", "bc"}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 2, (char *[]) {"x", "y"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 1, 2, a) == 0); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 2, 2, b) == 0); + first = cmt_map_metric_get(&gauge->opts, gauge->map, 2, a, CMT_FALSE); + second = cmt_map_metric_get(&gauge->opts, gauge->map, 2, b, CMT_FALSE); + TEST_ASSERT(first != NULL && second != NULL && first != second); + TEST_ASSERT(first->hash == second->hash); + first->otlp_metadata = cfl_kvlist_create(); + second->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(first->otlp_metadata != NULL && second->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(first->otlp_metadata, "flags", 1) == 0); + TEST_ASSERT(cfl_kvlist_insert_uint64(second->otlp_metadata, "flags", 0) == 0); + + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_install(ctx, gauge->map, internal, external); + cmt_map_metric_destroy(first); + + decoded = relay_round_trip(ctx, &buffer); + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(cfl_list_size(&gauge->map->metrics) == 1); + second = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + flags = cfl_kvlist_fetch(second->otlp_metadata, "flags"); + TEST_ASSERT(flags != NULL); + TEST_CHECK((flags->type == CFL_VARIANT_UINT && flags->data.as_uint64 == 0) || + (flags->type == CFL_VARIANT_INT && flags->data.as_int64 == 0)); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* One window of series "a" at time 100, as an OTLP point with its own + * start time. */ +static struct cmt *relay_window(uint64_t start, uint64_t flags, double value) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 1, (char *[]) {"site"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, value, 1, (char *[]) {"a"}) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, (char *[]) {"a"}, CMT_FALSE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_start_timestamp(sample, start); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", flags) == 0); + return ctx; +} + +/* Series "a" with point metadata nested beyond the decoder's depth limit, + * relayed by a released reader, optionally after changing its value at the + * same time. */ +static struct cmt *relay_unreadable(int change_value, char **buffer) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *inner; + struct cfl_kvlist *outer; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + struct cmt *decoded; + int depth; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 1, 0, CMT_FALSE); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, (char *[]) {"a"}, CMT_FALSE); + TEST_ASSERT(sample != NULL); + inner = cfl_kvlist_create(); + TEST_ASSERT(inner != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(inner, "leaf", "v") == 0); + for (depth = 0; depth < 40; depth++) { + outer = cfl_kvlist_create(); + TEST_ASSERT(outer != NULL); + TEST_ASSERT(cfl_kvlist_insert_kvlist(outer, "nested", inner) == 0); + inner = outer; + } + TEST_ASSERT(cfl_kvlist_insert_kvlist(sample->otlp_metadata, "nested", inner) == 0); + + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_install(ctx, gauge->map, internal, external); + if (change_value) { + TEST_ASSERT(cmt_gauge_set(gauge, 100, 99, 1, (char *[]) {"a"}) == 0); + } + decoded = relay_round_trip(ctx, buffer); + cmt_destroy(ctx); + return decoded; +} + +enum shared_key_sidecar { + SHARED_KEY_CURRENT, /* restored */ + SHARED_KEY_V230, /* no identifiers, point count changed: skipped */ + SHARED_KEY_NEW_FAMILY /* relay added a family: rejected */ +}; + +/* A released relay adds a series whose labels share the series hash of a + * recorded point, at the same time. That point's legacy entry, its only + * readable metadata, can belong to either: neither may take it, whether the + * native sidecar is restored, skipped or rejected. */ +static void check_shared_legacy_key(enum shared_key_sidecar sidecar) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *added; + struct cmt_metric *sample; + struct cfl_kvlist *inner; + struct cfl_kvlist *outer; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + cfl_sds_t wire; + char *buffer; + char *a[] = {"ab", "c"}; + char *b[] = {"a", "bc"}; + size_t index; + int depth; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 2, (char *[]) {"x", "y"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 1, 2, a) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 2, a, CMT_FALSE); + TEST_ASSERT(sample != NULL); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", 1) == 0); + inner = cfl_kvlist_create(); + TEST_ASSERT(inner != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(inner, "leaf", "v") == 0); + for (depth = 0; depth < 40; depth++) { + outer = cfl_kvlist_create(); + TEST_ASSERT(outer != NULL); + TEST_ASSERT(cfl_kvlist_insert_kvlist(outer, "nested", inner) == 0); + inner = outer; + } + TEST_ASSERT(cfl_kvlist_insert_kvlist(sample->otlp_metadata, "nested", inner) == 0); + + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + if (sidecar == SHARED_KEY_V230) { + relay_strip_point_ids(internal); + } + relay_install(ctx, gauge->map, internal, external); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 2, 2, b) == 0); + TEST_ASSERT(cmt_map_metric_get(&gauge->opts, gauge->map, 2, b, CMT_FALSE)->hash == + sample->hash); + if (sidecar == SHARED_KEY_NEW_FAMILY) { + added = cmt_gauge_create(ctx, "", "", "added", "added", 0, NULL); + TEST_ASSERT(added != NULL); + TEST_ASSERT(cmt_gauge_set(added, 100, 3, 0, NULL) == 0); + } + + decoded = relay_round_trip(ctx, &buffer); + wire = cmt_encode_opentelemetry_create(decoded); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "relay"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 2); + for (index = 0; index < metric->gauge->n_data_points; index++) { + TEST_CHECK_(metric->gauge->data_points[index]->flags == 0, + "sidecar %d: point %g takes no shared legacy flags", sidecar, + metric->gauge->data_points[index]->as_double); + } + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void test_otlp_restore_drops_shared_legacy_key(void) +{ + check_shared_legacy_key(SHARED_KEY_CURRENT); + check_shared_legacy_key(SHARED_KEY_V230); + check_shared_legacy_key(SHARED_KEY_NEW_FAMILY); +} + +/* Native point metadata that cannot be read back leaves the point its + * legacy entry, but only while its identity still matches: a changed value + * must not take the flags recorded for the old one. */ +static void test_otlp_restore_keeps_legacy_when_native_unreadable(void) +{ + struct cmt *decoded; + char *buffer; + uint32_t flags = 0; + + decoded = relay_unreadable(CMT_FALSE, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 1); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + + decoded = relay_unreadable(CMT_TRUE, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); +} + +/* A released relay changes the value of series "b" without changing its + * time. The metadata recorded for the old value must not describe the new + * one, from the sidecar or from the legacy tree. */ +static void test_otlp_restore_drops_changed_value(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 1, 1, CMT_FALSE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 1, (char *[]) {"b"}) == 0); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 1); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Two windows of one series differ only by start time. A released relay + * drops the first: its metadata must not move to the survivor. */ +static void test_otlp_restore_binds_points_by_start_time(void) +{ + struct cmt *ctx; + struct cmt *window; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_metric *first; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + window = relay_window(10, 1, 1.0); + TEST_ASSERT(cmt_cat(ctx, window) == 0); + cmt_destroy(window); + window = relay_window(20, 0, 2.0); + TEST_ASSERT(cmt_cat(ctx, window) == 0); + cmt_destroy(window); + + gauge = cfl_list_entry_first(&ctx->gauges, struct cmt_gauge, _head); + TEST_ASSERT(cfl_list_size(&gauge->map->metrics) == 2); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + first = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_ASSERT(cmt_metric_get_start_timestamp(first) == 10); + cmt_map_metric_destroy(first); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* A v2.3.0 sidecar binds by position. When a released relay removed a series, + * its points are not restored, and the legacy tree must still provide what + * released readers provide, even though family metadata was restored. */ +static void test_otlp_restore_keeps_unbound_legacy_points(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *result; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 0, 1, CMT_TRUE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_strip_point_ids(internal); + + relay_install(ctx, gauge->map, internal, external); + cmt_map_metric_destroy(cmt_map_metric_get(&gauge->opts, gauge->map, 1, + (char *[]) {"a"}, CMT_FALSE)); + + decoded = relay_round_trip(ctx, &buffer); + result = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_CHECK(result->map->otlp_metadata != NULL); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 1); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + TEST_LIST = { + {"otlp_restore_binds_points_by_identity", test_otlp_restore_binds_points_by_identity}, + {"otlp_restore_drops_stale_observation", test_otlp_restore_drops_stale_observation}, + {"otlp_restore_drops_changed_value", test_otlp_restore_drops_changed_value}, + {"otlp_restore_keeps_legacy_when_native_unreadable", test_otlp_restore_keeps_legacy_when_native_unreadable}, + {"otlp_restore_drops_shared_legacy_key", test_otlp_restore_drops_shared_legacy_key}, + {"otlp_restore_binds_points_by_start_time", test_otlp_restore_binds_points_by_start_time}, + {"otlp_restore_binds_points_by_label_values", test_otlp_restore_binds_points_by_label_values}, + {"otlp_restore_binds_reordered_points", test_otlp_restore_binds_reordered_points}, + {"otlp_restore_keeps_unbound_legacy_points", test_otlp_restore_keeps_unbound_legacy_points}, {"otlp_nested_empty_attributes_keep_point", test_otlp_nested_empty_attributes_keep_point}, + {"otlp_restore_preserves_legacy_only_family", test_otlp_restore_preserves_legacy_only_family}, {"otlp_unrepresentable_attribute_keeps_point", test_otlp_unrepresentable_attribute_keeps_point}, {"otlp_metadata_legacy_external_tree", test_otlp_metadata_legacy_external_tree}, {"otlp_metadata_deep_nesting", test_otlp_metadata_deep_nesting}, From 83fcce0201388271f7fa5a7f5f776732e1b2873b Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 22 Oct 2026 17:13:11 -0600 Subject: [PATCH 5/7] cat: preserve family semantics and validate contributors Fluent Bit processors rebuild contexts with cmt_cat(), which looks up destination families by name only. It: - failed the whole context when an OTLP metric's points carried different attribute key sets - turned delta sums and histograms into cumulative ones and non-monotonic sums into counters - dropped the unit - added a zero point to families that had none Samples with fewer values than the label keys are accepted from a family with the same keys, the shape OTLP points with partial attributes take. Any other label count mismatch fails before the destination changes. Unit, temporality and monotonicity are taken from contributors while a family holds no samples and never changed afterwards. A contributor with samples and a conflicting temporality, monotonicity or explicit unit fails before the destination changes, since values are never converted. Unspecified temporality, written by releases that did not record it, and an empty unit are compatible with any. Signed-off-by: Eduardo Silva --- src/cmt_cat.c | 226 +++++++++++++++++++++++++++++++++++++++++++------- tests/cat.c | 201 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 395 insertions(+), 32 deletions(-) diff --git a/src/cmt_cat.c b/src/cmt_cat.c index dbcd3bf5..aba564ad 100644 --- a/src/cmt_cat.c +++ b/src/cmt_cat.c @@ -622,6 +622,20 @@ static struct cmt_metric *get_point_sample(struct cmt_opts *opts, struct cmt_map struct cfl_list *head; struct cmt_metric *metric; + if (count > 0 && count != dst->label_count) { + /* OTLP points of one metric may carry fewer attributes than the + * family has label keys. The index only holds complete label sets, + * so such samples are matched by value and appended. */ + cfl_list_foreach(head, &dst->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (sample_labels_match(metric, count, labels) && + (src->otlp_metadata == NULL || same_data_point(metric, src))) { + return metric; + } + } + return append_point_sample(dst, src, count, labels); + } + if (count == 0) { /* Label-less points always merge into the static metric. */ return cmt_map_metric_get(opts, dst, 0, NULL, CMT_TRUE); @@ -648,6 +662,87 @@ static struct cmt_metric *get_point_sample(struct cmt_opts *opts, struct cmt_map return append_point_sample(dst, src, count, labels); } +/* + * Unit and static sample state describe a family. They are set when cmt_cat() + * creates it, and again for each contributor while it holds no samples, then + * never changed: later contributors cannot reinterpret samples already + * merged. Label-less families are created with their static sample set: a + * new destination must not report a sample its source never had. + */ +static int init_family(struct cmt_map *dst, struct cmt_map *family, + struct cmt_map *points) +{ + if (dst->unit != NULL) { + cfl_sds_destroy(dst->unit); + dst->unit = NULL; + } + if (family->unit != NULL) { + dst->unit = cfl_sds_create_len(family->unit, cfl_sds_len(family->unit)); + if (dst->unit == NULL) { + return -1; + } + } + if (!points->metric_static_set) { + dst->metric_static_set = CMT_FALSE; + } + return 0; +} + +static int map_has_samples(struct cmt_map *map) +{ + return map->metric_static_set || cfl_list_size(&map->metrics) > 0; +} + +/* Unspecified temporality (written by releases that did not record it) is + * compatible with either. */ +static int aggregation_compatible(int dst, int src) +{ + return dst == src || + dst == CMT_AGGREGATION_TYPE_UNSPECIFIED || + src == CMT_AGGREGATION_TYPE_UNSPECIFIED; +} + +/* An empty unit is unspecified and compatible with any. Values are never + * converted, so two explicit units must be equal. */ +static int units_compatible(struct cmt_map *dst, struct cmt_map *src) +{ + if (dst->unit == NULL || src->unit == NULL || + cfl_sds_len(dst->unit) == 0 || cfl_sds_len(src->unit) == 0) { + return CMT_TRUE; + } + return cfl_sds_len(dst->unit) == cfl_sds_len(src->unit) && + memcmp(dst->unit, src->unit, cfl_sds_len(src->unit)) == 0; +} + +static int label_keys_match(struct cmt_map *left, + struct cmt_map *right) +{ + struct cfl_list *left_head; + struct cfl_list *right_head; + struct cmt_map_label *left_label; + struct cmt_map_label *right_label; + + left_head = left->label_keys.next; + right_head = right->label_keys.next; + + while (left_head != &left->label_keys && + right_head != &right->label_keys) { + left_label = cfl_list_entry(left_head, struct cmt_map_label, _head); + right_label = cfl_list_entry(right_head, struct cmt_map_label, _head); + + if (left_label->name == NULL || right_label->name == NULL || + strcmp(left_label->name, right_label->name) != 0) { + return CMT_FALSE; + } + + left_head = left_head->next; + right_head = right_head->next; + } + + return left_head == &left->label_keys && + right_head == &right->label_keys; +} + int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map *src) { int c; @@ -660,6 +755,20 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map struct cmt_summary *summary; struct cmt_histogram *histogram_src; struct cmt_histogram *histogram_dst; + int same_keys; + + /* Only OTLP points of the same family carry fewer values than its label + * keys; any other mismatch has no meaning here. Check every sample before + * changing the destination. */ + same_keys = label_keys_match(dst, src); + cfl_list_foreach(head, &src->metrics) { + metric_src = cfl_list_entry(head, struct cmt_metric, _head); + c = cfl_list_size(&metric_src->labels); + if (c > 0 && c != dst->label_count && + (c > dst->label_count || !same_keys)) { + return -1; + } + } if (src->otlp_metadata) { metadata = cmt_otlp_metadata_clone(src->otlp_metadata); @@ -716,7 +825,7 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } c = cfl_list_size(&metric_src->labels); - if (metric_src->otlp_metadata != NULL) { + if (metric_src->otlp_metadata != NULL || (c > 0 && c != dst->label_count)) { metric_dst = get_point_sample(opts, dst, metric_src, c, labels); } else { @@ -872,40 +981,12 @@ static struct cmt_summary *summary_lookup(struct cmt *cmt, struct cmt_opts *opts return NULL; } -static int summary_label_keys_match(struct cmt_map *left, - struct cmt_map *right) -{ - struct cfl_list *left_head; - struct cfl_list *right_head; - struct cmt_map_label *left_label; - struct cmt_map_label *right_label; - - left_head = left->label_keys.next; - right_head = right->label_keys.next; - - while (left_head != &left->label_keys && - right_head != &right->label_keys) { - left_label = cfl_list_entry(left_head, struct cmt_map_label, _head); - right_label = cfl_list_entry(right_head, struct cmt_map_label, _head); - - if (left_label->name == NULL || right_label->name == NULL || - strcmp(left_label->name, right_label->name) != 0) { - return CMT_FALSE; - } - - left_head = left_head->next; - right_head = right_head->next; - } - - return left_head == &left->label_keys && - right_head == &right->label_keys; -} - int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, struct cmt_map *filtered_map) { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_counter *c; @@ -925,6 +1006,7 @@ int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -932,6 +1014,22 @@ int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, return -1; } + /* Temporality and monotonicity are family semantics: a contributor + * with samples must not change how existing samples are read. */ + if (created || !map_has_samples(c->map)) { + if (init_family(c->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + c->aggregation_type = counter->aggregation_type; + c->allow_reset = counter->allow_reset; + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + (!aggregation_compatible(c->aggregation_type, counter->aggregation_type) || + c->allow_reset != counter->allow_reset || + !units_compatible(c->map, map))) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&c->opts, c->map, filtered_map); if (ret == -1) { @@ -953,6 +1051,7 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_gauge *g; @@ -972,6 +1071,7 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -979,6 +1079,16 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, return -1; } + if (created || !map_has_samples(g->map)) { + if (init_family(g->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + !units_compatible(g->map, map)) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&g->opts, g->map, filtered_map); if (ret == -1) { @@ -1000,6 +1110,7 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_untyped *u; @@ -1019,6 +1130,7 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -1026,6 +1138,16 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, return -1; } + if (created || !map_has_samples(u->map)) { + if (init_family(u->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + !units_compatible(u->map, map)) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&u->opts, u->map, filtered_map); if (ret == -1) { @@ -1047,6 +1169,7 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_histogram *hist; @@ -1078,6 +1201,7 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, opts->name, opts->description, buckets, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -1085,6 +1209,19 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, return -1; } + if (created || !map_has_samples(hist->map)) { + if (init_family(hist->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + hist->aggregation_type = histogram->aggregation_type; + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + (!aggregation_compatible(hist->aggregation_type, + histogram->aggregation_type) || + !units_compatible(hist->map, map))) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&hist->opts, hist->map, filtered_map); if (ret == -1) { @@ -1107,6 +1244,7 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, size_t i; int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_summary *sum; @@ -1121,7 +1259,7 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, sum = summary_lookup(cmt, opts); if (sum != NULL) { - if (!summary_label_keys_match(sum->map, map)) { + if (!label_keys_match(sum->map, map)) { free(labels); return -1; } @@ -1156,6 +1294,7 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, summary->quantiles_count, quantiles, map->label_count, labels); + created = CMT_TRUE; free(quantiles); } @@ -1164,6 +1303,16 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, return -1; } + if (created || !map_has_samples(sum->map)) { + if (init_family(sum->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + !units_compatible(sum->map, map)) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&sum->opts, sum->map, filtered_map); if (ret == -1) { @@ -1185,6 +1334,7 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_exp_histogram *eh; @@ -1203,6 +1353,7 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -1210,7 +1361,18 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr return -1; } - eh->aggregation_type = exp_histogram->aggregation_type; + if (created || !map_has_samples(eh->map)) { + if (init_family(eh->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + eh->aggregation_type = exp_histogram->aggregation_type; + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + (!aggregation_compatible(eh->aggregation_type, + exp_histogram->aggregation_type) || + !units_compatible(eh->map, map))) { + return -1; + } if (filtered_map != NULL) { ret = cmt_cat_copy_map(&eh->opts, eh->map, filtered_map); diff --git a/tests/cat.c b/tests/cat.c index f27671aa..80d48322 100644 --- a/tests/cat.c +++ b/tests/cat.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "cmt_tests.h" @@ -1463,7 +1464,207 @@ void test_cat_otlp_metadata_distinct_points_kept() cmt_destroy(final); } +/* A family with more label keys than the destination cannot be merged: the + * destination must stay encodable and keep its samples. */ +void test_cat_rejects_larger_label_schema() +{ + struct cmt *a; + struct cmt *b; + struct cmt *dst; + struct cmt_gauge *gauge_a; + struct cmt_gauge *gauge_b; + struct cmt_gauge *out; + cfl_sds_t wire; + + a = cmt_create(); + b = cmt_create(); + dst = cmt_create(); + TEST_ASSERT(a != NULL && b != NULL && dst != NULL); + gauge_a = cmt_gauge_create(a, "", "", "g", "g", 1, (char *[]) {"a"}); + gauge_b = cmt_gauge_create(b, "", "", "g", "g", 2, (char *[]) {"a", "b"}); + TEST_ASSERT(gauge_a != NULL && gauge_b != NULL); + TEST_ASSERT(cmt_gauge_set(gauge_a, 100, 1, 1, (char *[]) {"x"}) == 0); + TEST_ASSERT(cmt_gauge_set(gauge_b, 200, 2, 2, (char *[]) {"x", "y"}) == 0); + + TEST_CHECK(cmt_cat(dst, a) == 0); + TEST_CHECK(cmt_cat(dst, b) == -1); + + out = cfl_list_entry_first(&dst->gauges, struct cmt_gauge, _head); + TEST_CHECK(out->map->label_count == 1); + TEST_CHECK(cfl_list_size(&out->map->metrics) == 1); + wire = cmt_encode_opentelemetry_create(dst); + TEST_CHECK(wire != NULL); + if (wire != NULL) { + cmt_encode_opentelemetry_destroy(wire); + } + + cmt_destroy(a); + cmt_destroy(b); + cmt_destroy(dst); +} + +static struct cmt *semantics_counter(int aggregation_type, int allow_reset, int with_sample, + char *unit) +{ + struct cmt *ctx; + struct cmt_counter *counter; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + counter = cmt_counter_create(ctx, "", "", "requests", "requests", 1, (char *[]) {"site"}); + TEST_ASSERT(counter != NULL); + counter->aggregation_type = aggregation_type; + counter->allow_reset = allow_reset; + if (unit != NULL) { + counter->map->unit = cfl_sds_create(unit); + TEST_ASSERT(counter->map->unit != NULL); + } + if (with_sample) { + TEST_ASSERT(cmt_counter_set(counter, 100, 5, 1, (char *[]) {"b"}) == 0); + } + return ctx; +} + +static struct cmt *semantics_histogram(int aggregation_type) +{ + struct cmt *ctx; + struct cmt_histogram *histogram; + uint64_t buckets[] = {1, 2}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + histogram = cmt_histogram_create(ctx, "", "", "latency", "latency", + cmt_histogram_buckets_create(1, 1.0), 0, NULL); + TEST_ASSERT(histogram != NULL); + histogram->aggregation_type = aggregation_type; + TEST_ASSERT(cmt_histogram_set_default(histogram, 100, buckets, 1.5, 2, 0, NULL) == 0); + return ctx; +} + +/* Later contributors must not change how samples already in an existing + * family are read: an empty one is a no-op and a conflicting one fails + * without changes. */ +void test_cat_keeps_existing_family_semantics() +{ + struct cmt *dst; + struct cmt *src; + struct cmt_counter *counter; + struct cmt_histogram *histogram; + + dst = cmt_create(); + TEST_ASSERT(dst != NULL); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, NULL); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + counter = cfl_list_entry_first(&dst->counters, struct cmt_counter, _head); + + /* empty contributors: delta, non-monotonic, with a unit */ + src = semantics_counter(CMT_AGGREGATION_TYPE_DELTA, CMT_TRUE, CMT_FALSE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + TEST_CHECK(counter->allow_reset == CMT_FALSE); + TEST_CHECK(counter->map->unit == NULL); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + + /* contributors with samples and conflicting semantics */ + src = semantics_counter(CMT_AGGREGATION_TYPE_DELTA, CMT_FALSE, CMT_TRUE, NULL); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_TRUE, CMT_TRUE, NULL); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + TEST_CHECK(counter->allow_reset == CMT_FALSE); + TEST_CHECK(cmt_metric_get_value(cfl_list_entry_first(&counter->map->metrics, + struct cmt_metric, _head)) == 5); + cmt_destroy(dst); + + /* unspecified temporality (Fluent Bit 5.0.x histograms) merges with + * either and keeps the destination's */ + dst = semantics_histogram(CMT_AGGREGATION_TYPE_CUMULATIVE); + src = semantics_histogram(CMT_AGGREGATION_TYPE_UNSPECIFIED); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + histogram = cfl_list_entry_first(&dst->histograms, struct cmt_histogram, _head); + TEST_CHECK(histogram->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + src = semantics_histogram(CMT_AGGREGATION_TYPE_DELTA); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + TEST_CHECK(histogram->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + cmt_destroy(dst); +} + +/* A destination family that holds no samples has nothing to reinterpret: + * it takes the semantics of the first contributor that brings samples. */ +void test_cat_empty_family_adopts_semantics() +{ + struct cmt *dst; + struct cmt *src; + struct cmt_counter *counter; + + dst = cmt_create(); + TEST_ASSERT(dst != NULL); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_FALSE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + counter = cfl_list_entry_first(&dst->counters, struct cmt_counter, _head); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 0); + + src = semantics_counter(CMT_AGGREGATION_TYPE_DELTA, CMT_TRUE, CMT_TRUE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); + TEST_CHECK(counter->allow_reset == CMT_TRUE); + TEST_CHECK(counter->map->unit != NULL && strcmp(counter->map->unit, "ms") == 0); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + + /* once it holds samples, its semantics stay */ + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); + cmt_destroy(dst); +} + +/* Values are never converted, so families with different explicit units + * cannot be merged. An empty unit is unspecified. */ +void test_cat_rejects_conflicting_units() +{ + struct cmt *dst; + struct cmt *src; + struct cmt_counter *counter; + + dst = cmt_create(); + TEST_ASSERT(dst != NULL); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + counter = cfl_list_entry_first(&dst->counters, struct cmt_counter, _head); + + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_FALSE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, ""); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + + TEST_CHECK(strcmp(counter->map->unit, "s") == 0); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + cmt_destroy(dst); +} + TEST_LIST = { + {"cat_rejects_larger_label_schema", test_cat_rejects_larger_label_schema}, + {"cat_keeps_existing_family_semantics", test_cat_keeps_existing_family_semantics}, + {"cat_rejects_conflicting_units", test_cat_rejects_conflicting_units}, + {"cat_empty_family_adopts_semantics", test_cat_empty_family_adopts_semantics}, {"cat_clears_stale_otlp_point_metadata", test_cat_clears_stale_otlp_point_metadata}, {"cat_otlp_metadata_distinct_points_kept", test_cat_otlp_metadata_distinct_points_kept}, {"cat_otlp_metadata_samples_merge_by_labels", test_cat_otlp_metadata_samples_merge_by_labels}, From ad80cf561c918704340877774d7896e1bd48fbc9 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 22 Oct 2026 17:13:11 -0600 Subject: [PATCH 6/7] tests: check OTLP data points through every pipeline Compare every OTLP data point field after direct, MessagePack chunk, two-hop forward and cmt_cat() pipelines: all metric types, integer and float extremes, NaN, infinities and staleness markers, typed attributes, resources and scopes, exemplars, malformed requests and empty containers, plus points that differ only by attribute type relayed by a released reader. Signed-off-by: Eduardo Silva --- tests/CMakeLists.txt | 1 + tests/otlp_pipeline.c | 1794 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1795 insertions(+) create mode 100644 tests/otlp_pipeline.c diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5b9c9bc3..39e031c0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -20,6 +20,7 @@ set(UNIT_TESTS_FILES msgpack_temporality.c format_conversion.c expire.c + otlp_pipeline.c ) if (CMT_BUILD_PROMETHEUS_TEXT_DECODER) diff --git a/tests/otlp_pipeline.c b/tests/otlp_pipeline.c new file mode 100644 index 00000000..0dba4c2c --- /dev/null +++ b/tests/otlp_pipeline.c @@ -0,0 +1,1794 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * OTLP traffic through the transformations Fluent Bit applies to it: + * + * direct : in_opentelemetry -> out_opentelemetry in one process + * chunk : OTLP -> MessagePack chunk -> OTLP (storage, output processors) + * forward : two MessagePack hops (agent -> forward -> aggregator -> OTLP) + * cat : chunk decoded and merged with cmt_cat (batching, processors) + * + * Every data point of the input request is reduced to a canonical line that + * includes its resource, scope, metric and point fields with exact bit + * patterns for doubles and typed, order independent attributes. A pipeline is + * lossless when the output request produces the same multiset of lines. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "cmt_tests.h" + +/* Builder and canonicalization allocations: abort rather than flood the + * check log, these are test infrastructure, not behavior under test. */ +static void *require(void *pointer) +{ + if (pointer == NULL) { + abort(); + } + return pointer; +} + +typedef Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest otlp_request; +typedef Opentelemetry__Proto__Metrics__V1__ResourceMetrics otlp_resource_metrics; +typedef Opentelemetry__Proto__Metrics__V1__ScopeMetrics otlp_scope_metrics; +typedef Opentelemetry__Proto__Metrics__V1__Metric otlp_metric; +typedef Opentelemetry__Proto__Metrics__V1__NumberDataPoint otlp_number_point; +typedef Opentelemetry__Proto__Metrics__V1__HistogramDataPoint otlp_histogram_point; +typedef Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint otlp_exp_point; +typedef Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets otlp_exp_buckets; +typedef Opentelemetry__Proto__Metrics__V1__SummaryDataPoint otlp_summary_point; +typedef Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile otlp_quantile; +typedef Opentelemetry__Proto__Metrics__V1__Exemplar otlp_exemplar; +typedef Opentelemetry__Proto__Common__V1__KeyValue otlp_kv; +typedef Opentelemetry__Proto__Common__V1__AnyValue otlp_any; + +#define TEMPORALITY_DELTA \ + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_DELTA +#define TEMPORALITY_CUMULATIVE \ + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE + +enum pipeline { + PIPELINE_DIRECT = 0, + PIPELINE_CHUNK, + PIPELINE_FORWARD, + PIPELINE_CAT, + PIPELINE_COUNT +}; + +static const char *pipeline_names[] = {"direct", "chunk", "forward", "cat"}; + +/* Every builder allocation is released at once after a case. */ +static void **allocations; +static size_t allocation_count; +static size_t allocation_size; + +static void *test_alloc(size_t size) +{ + void **grown; + void *memory; + + if (allocation_count == allocation_size) { + allocation_size = allocation_size ? allocation_size * 2 : 64; + grown = require(realloc(allocations, allocation_size * sizeof(*allocations))); + allocations = grown; + } + memory = require(calloc(1, size ? size : 1)); + allocations[allocation_count++] = memory; + return memory; +} + +static void test_alloc_release(void) +{ + size_t index; + + for (index = 0; index < allocation_count; index++) { + free(allocations[index]); + } + free(allocations); + allocations = NULL; + allocation_count = 0; + allocation_size = 0; +} + +static void **pointer_list(size_t count, va_list arguments) +{ + void **list; + size_t index; + + list = test_alloc(sizeof(void *) * count); + for (index = 0; index < count; index++) { + list[index] = va_arg(arguments, void *); + } + return list; +} + +/* AnyValue and KeyValue builders */ + +static otlp_any *any_string(const char *value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE; + any->string_value = (char *) value; + return any; +} + +static otlp_any *any_int(int64_t value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE; + any->int_value = value; + return any; +} + +static otlp_any *any_double(double value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE; + any->double_value = value; + return any; +} + +static otlp_any *any_bool(int value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE; + any->bool_value = value; + return any; +} + +static otlp_any *any_bytes(const char *value, size_t length) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE; + any->bytes_value.data = (uint8_t *) value; + any->bytes_value.len = length; + return any; +} + +static otlp_any *any_array(size_t count, ...) +{ + va_list arguments; + otlp_any *any = test_alloc(sizeof(*any)); + Opentelemetry__Proto__Common__V1__ArrayValue *array = test_alloc(sizeof(*array)); + + opentelemetry__proto__common__v1__any_value__init(any); + opentelemetry__proto__common__v1__array_value__init(array); + va_start(arguments, count); + array->values = (otlp_any **) pointer_list(count, arguments); + va_end(arguments); + array->n_values = count; + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE; + any->array_value = array; + return any; +} + +static otlp_any *any_kvlist(size_t count, ...) +{ + va_list arguments; + otlp_any *any = test_alloc(sizeof(*any)); + Opentelemetry__Proto__Common__V1__KeyValueList *list = test_alloc(sizeof(*list)); + + opentelemetry__proto__common__v1__any_value__init(any); + opentelemetry__proto__common__v1__key_value_list__init(list); + va_start(arguments, count); + list->values = (otlp_kv **) pointer_list(count, arguments); + va_end(arguments); + list->n_values = count; + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE; + any->kvlist_value = list; + return any; +} + +static otlp_kv *kv(const char *key, otlp_any *value) +{ + otlp_kv *pair = test_alloc(sizeof(*pair)); + + opentelemetry__proto__common__v1__key_value__init(pair); + pair->key = (char *) key; + pair->value = value; + return pair; +} + +static otlp_kv **kv_list(size_t count, ...) +{ + va_list arguments; + otlp_kv **list; + + va_start(arguments, count); + list = (otlp_kv **) pointer_list(count, arguments); + va_end(arguments); + return list; +} + +/* Data point builders */ + +static otlp_number_point *number_int(uint64_t start, uint64_t time, int64_t value, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_number_point *point = test_alloc(sizeof(*point)); + + opentelemetry__proto__metrics__v1__number_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + point->as_int = value; + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +static otlp_number_point *number_double(uint64_t start, uint64_t time, double value, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_number_point *point = number_int(start, time, 0, attribute_count, attributes); + + point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + point->as_double = value; + return point; +} + +static otlp_exemplar *exemplar_double(uint64_t time, double value, const char *trace_id, + const char *span_id, size_t attribute_count, + otlp_kv **attributes) +{ + otlp_exemplar *exemplar = test_alloc(sizeof(*exemplar)); + + opentelemetry__proto__metrics__v1__exemplar__init(exemplar); + exemplar->time_unix_nano = time; + exemplar->value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_DOUBLE; + exemplar->as_double = value; + if (trace_id != NULL) { + exemplar->trace_id.data = (uint8_t *) trace_id; + exemplar->trace_id.len = 16; + } + if (span_id != NULL) { + exemplar->span_id.data = (uint8_t *) span_id; + exemplar->span_id.len = 8; + } + exemplar->n_filtered_attributes = attribute_count; + exemplar->filtered_attributes = attributes; + return exemplar; +} + +static otlp_exemplar *exemplar_int(uint64_t time, int64_t value) +{ + otlp_exemplar *exemplar = exemplar_double(time, 0, NULL, NULL, 0, NULL); + + exemplar->value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_INT; + exemplar->as_int = value; + return exemplar; +} + +static otlp_exemplar **exemplar_list(size_t count, ...) +{ + va_list arguments; + otlp_exemplar **list; + + va_start(arguments, count); + list = (otlp_exemplar **) pointer_list(count, arguments); + va_end(arguments); + return list; +} + +static otlp_histogram_point *histogram_point(uint64_t start, uint64_t time, + size_t bound_count, const double *bounds, + const uint64_t *counts, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_histogram_point *point = test_alloc(sizeof(*point)); + size_t index; + + opentelemetry__proto__metrics__v1__histogram_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->n_explicit_bounds = bound_count; + point->explicit_bounds = test_alloc(sizeof(double) * (bound_count + 1)); + if (bound_count > 0) { + memcpy(point->explicit_bounds, bounds, sizeof(double) * bound_count); + } + point->n_bucket_counts = bound_count + 1; + point->bucket_counts = test_alloc(sizeof(uint64_t) * (bound_count + 1)); + memcpy(point->bucket_counts, counts, sizeof(uint64_t) * (bound_count + 1)); + for (index = 0; index <= bound_count; index++) { + point->count += counts[index]; + } + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +static otlp_exp_buckets *exp_buckets(int32_t offset, size_t count, const uint64_t *counts) +{ + otlp_exp_buckets *buckets = test_alloc(sizeof(*buckets)); + + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(buckets); + buckets->offset = offset; + buckets->n_bucket_counts = count; + buckets->bucket_counts = test_alloc(sizeof(uint64_t) * (count + 1)); + memcpy(buckets->bucket_counts, counts, sizeof(uint64_t) * count); + return buckets; +} + +static otlp_exp_point *exp_point(uint64_t start, uint64_t time, int32_t scale, + uint64_t zero_count, double zero_threshold, + otlp_exp_buckets *positive, otlp_exp_buckets *negative, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_exp_point *point = test_alloc(sizeof(*point)); + size_t index; + + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->scale = scale; + point->zero_count = zero_count; + point->zero_threshold = zero_threshold; + point->positive = positive; + point->negative = negative; + point->count = zero_count; + for (index = 0; positive != NULL && index < positive->n_bucket_counts; index++) { + point->count += positive->bucket_counts[index]; + } + for (index = 0; negative != NULL && index < negative->n_bucket_counts; index++) { + point->count += negative->bucket_counts[index]; + } + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +static otlp_summary_point *summary_point(uint64_t start, uint64_t time, uint64_t count, + double sum, size_t quantile_count, + const double *quantiles, const double *values, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_summary_point *point = test_alloc(sizeof(*point)); + size_t index; + + opentelemetry__proto__metrics__v1__summary_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->count = count; + point->sum = sum; + point->n_quantile_values = quantile_count; + point->quantile_values = test_alloc(sizeof(otlp_quantile *) * (quantile_count + 1)); + for (index = 0; index < quantile_count; index++) { + point->quantile_values[index] = test_alloc(sizeof(otlp_quantile)); + opentelemetry__proto__metrics__v1__summary_data_point__value_at_quantile__init( + point->quantile_values[index]); + point->quantile_values[index]->quantile = quantiles[index]; + point->quantile_values[index]->value = values[index]; + } + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +/* Metric builders: points are passed as a count and a variadic list. */ + +static otlp_metric *metric_base(const char *name, const char *description, const char *unit) +{ + otlp_metric *metric = test_alloc(sizeof(*metric)); + + opentelemetry__proto__metrics__v1__metric__init(metric); + metric->name = (char *) name; + metric->description = (char *) (description ? description : ""); + metric->unit = (char *) (unit ? unit : ""); + return metric; +} + +static otlp_metric *metric_gauge(const char *name, const char *description, const char *unit, + size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, description, unit); + Opentelemetry__Proto__Metrics__V1__Gauge *gauge = test_alloc(sizeof(*gauge)); + + opentelemetry__proto__metrics__v1__gauge__init(gauge); + va_start(arguments, count); + gauge->data_points = (otlp_number_point **) pointer_list(count, arguments); + va_end(arguments); + gauge->n_data_points = count; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE; + metric->gauge = gauge; + return metric; +} + +static otlp_metric *metric_sum(const char *name, const char *description, const char *unit, + int temporality, int monotonic, size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, description, unit); + Opentelemetry__Proto__Metrics__V1__Sum *sum = test_alloc(sizeof(*sum)); + + opentelemetry__proto__metrics__v1__sum__init(sum); + va_start(arguments, count); + sum->data_points = (otlp_number_point **) pointer_list(count, arguments); + va_end(arguments); + sum->n_data_points = count; + sum->aggregation_temporality = temporality; + sum->is_monotonic = monotonic; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM; + metric->sum = sum; + return metric; +} + +static otlp_metric *metric_histogram(const char *name, const char *unit, int temporality, + size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, "histogram", unit); + Opentelemetry__Proto__Metrics__V1__Histogram *histogram = test_alloc(sizeof(*histogram)); + + opentelemetry__proto__metrics__v1__histogram__init(histogram); + va_start(arguments, count); + histogram->data_points = (otlp_histogram_point **) pointer_list(count, arguments); + va_end(arguments); + histogram->n_data_points = count; + histogram->aggregation_temporality = temporality; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM; + metric->histogram = histogram; + return metric; +} + +static otlp_metric *metric_exp_histogram(const char *name, int temporality, size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, "exponential histogram", "s"); + Opentelemetry__Proto__Metrics__V1__ExponentialHistogram *histogram = + test_alloc(sizeof(*histogram)); + + opentelemetry__proto__metrics__v1__exponential_histogram__init(histogram); + va_start(arguments, count); + histogram->data_points = (otlp_exp_point **) pointer_list(count, arguments); + va_end(arguments); + histogram->n_data_points = count; + histogram->aggregation_temporality = temporality; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM; + metric->exponential_histogram = histogram; + return metric; +} + +static otlp_metric *metric_summary(const char *name, size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, "summary", "ms"); + Opentelemetry__Proto__Metrics__V1__Summary *summary = test_alloc(sizeof(*summary)); + + opentelemetry__proto__metrics__v1__summary__init(summary); + va_start(arguments, count); + summary->data_points = (otlp_summary_point **) pointer_list(count, arguments); + va_end(arguments); + summary->n_data_points = count; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUMMARY; + metric->summary = summary; + return metric; +} + +static otlp_scope_metrics *scope(const char *name, const char *version, + size_t attribute_count, otlp_kv **attributes, + size_t count, ...) +{ + va_list arguments; + otlp_scope_metrics *scope_metrics = test_alloc(sizeof(*scope_metrics)); + Opentelemetry__Proto__Common__V1__InstrumentationScope *instrumentation; + + opentelemetry__proto__metrics__v1__scope_metrics__init(scope_metrics); + if (name != NULL) { + instrumentation = test_alloc(sizeof(*instrumentation)); + opentelemetry__proto__common__v1__instrumentation_scope__init(instrumentation); + instrumentation->name = (char *) name; + instrumentation->version = (char *) (version ? version : ""); + instrumentation->n_attributes = attribute_count; + instrumentation->attributes = attributes; + scope_metrics->scope = instrumentation; + } + va_start(arguments, count); + scope_metrics->metrics = (otlp_metric **) pointer_list(count, arguments); + va_end(arguments); + scope_metrics->n_metrics = count; + return scope_metrics; +} + +static otlp_resource_metrics *resource(size_t attribute_count, otlp_kv **attributes, + size_t count, ...) +{ + va_list arguments; + otlp_resource_metrics *resource_metrics = test_alloc(sizeof(*resource_metrics)); + Opentelemetry__Proto__Resource__V1__Resource *resource_value = + test_alloc(sizeof(*resource_value)); + + opentelemetry__proto__metrics__v1__resource_metrics__init(resource_metrics); + opentelemetry__proto__resource__v1__resource__init(resource_value); + resource_value->n_attributes = attribute_count; + resource_value->attributes = attributes; + resource_metrics->resource = resource_value; + va_start(arguments, count); + resource_metrics->scope_metrics = (otlp_scope_metrics **) pointer_list(count, arguments); + va_end(arguments); + resource_metrics->n_scope_metrics = count; + return resource_metrics; +} + +static otlp_request *request(size_t count, ...) +{ + va_list arguments; + otlp_request *value = test_alloc(sizeof(*value)); + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(value); + va_start(arguments, count); + value->resource_metrics = (otlp_resource_metrics **) pointer_list(count, arguments); + va_end(arguments); + value->n_resource_metrics = count; + return value; +} + +/* Canonical form */ + +/* cmt_cat merges metric families only; context metadata (resource, scope) + * is not merged by design, so the cat pipeline compares families alone. */ +static int canon_include_context = CMT_TRUE; + +struct lines { + cfl_sds_t *entries; + size_t count; + size_t size; +}; + +static void lines_add(struct lines *lines, cfl_sds_t line) +{ + cfl_sds_t *grown; + + if (lines->count == lines->size) { + lines->size = lines->size ? lines->size * 2 : 16; + grown = require(realloc(lines->entries, lines->size * sizeof(*grown))); + lines->entries = grown; + } + lines->entries[lines->count++] = line; +} + +static void lines_destroy(struct lines *lines) +{ + size_t index; + + for (index = 0; index < lines->count; index++) { + cfl_sds_destroy(lines->entries[index]); + } + free(lines->entries); + memset(lines, 0, sizeof(*lines)); +} + +static cfl_sds_t sds_printf(cfl_sds_t buffer, const char *format, ...) +{ + va_list arguments; + char stack[512]; + int length; + + va_start(arguments, format); + length = vsnprintf(stack, sizeof(stack), format, arguments); + va_end(arguments); + if (length < 0 || (size_t) length >= sizeof(stack)) { + abort(); + } + return require(cfl_sds_cat(buffer, stack, length)); +} + +static cfl_sds_t canon_double(cfl_sds_t buffer, double value) +{ + uint64_t bits; + + memcpy(&bits, &value, sizeof(bits)); + return sds_printf(buffer, "%016" PRIx64, bits); +} + +static cfl_sds_t canon_bytes(cfl_sds_t buffer, const uint8_t *data, size_t length) +{ + size_t index; + + for (index = 0; index < length; index++) { + buffer = sds_printf(buffer, "%02x", data[index]); + } + return buffer; +} + +static int compare_strings(const void *left, const void *right) +{ + return strcmp(*(const char **) left, *(const char **) right); +} + +static cfl_sds_t canon_attributes(cfl_sds_t buffer, size_t count, otlp_kv **attributes); + +static cfl_sds_t canon_any(cfl_sds_t buffer, otlp_any *value) +{ + size_t index; + + if (value == NULL) { + return cfl_sds_cat(buffer, "none", 4); + } + switch (value->value_case) { + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: + buffer = cfl_sds_cat(buffer, "s:", 2); + return cfl_sds_cat(buffer, value->string_value, strlen(value->string_value)); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: + return sds_printf(buffer, "i:%" PRId64, value->int_value); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: + buffer = cfl_sds_cat(buffer, "d:", 2); + return canon_double(buffer, value->double_value); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: + return sds_printf(buffer, "b:%d", value->bool_value ? 1 : 0); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: + buffer = cfl_sds_cat(buffer, "x:", 2); + return canon_bytes(buffer, value->bytes_value.data, value->bytes_value.len); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: + buffer = cfl_sds_cat(buffer, "[", 1); + for (index = 0; index < value->array_value->n_values; index++) { + buffer = canon_any(buffer, value->array_value->values[index]); + buffer = cfl_sds_cat(buffer, ",", 1); + } + return cfl_sds_cat(buffer, "]", 1); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: + return canon_attributes(buffer, value->kvlist_value->n_values, + value->kvlist_value->values); + default: + return cfl_sds_cat(buffer, "unset", 5); + } +} + +/* Attributes are a set in OTLP: canonicalize independently of order. */ +static cfl_sds_t canon_attributes(cfl_sds_t buffer, size_t count, otlp_kv **attributes) +{ + cfl_sds_t *entries; + size_t index; + + entries = require(calloc(count + 1, sizeof(*entries))); + for (index = 0; index < count; index++) { + entries[index] = cfl_sds_create(attributes[index]->key); + entries[index] = cfl_sds_cat(entries[index], "=", 1); + entries[index] = canon_any(entries[index], attributes[index]->value); + } + qsort(entries, count, sizeof(*entries), compare_strings); + buffer = cfl_sds_cat(buffer, "{", 1); + for (index = 0; index < count; index++) { + buffer = cfl_sds_cat(buffer, entries[index], cfl_sds_len(entries[index])); + buffer = cfl_sds_cat(buffer, ";", 1); + cfl_sds_destroy(entries[index]); + } + free(entries); + return cfl_sds_cat(buffer, "}", 1); +} + +static cfl_sds_t canon_exemplars(cfl_sds_t buffer, size_t count, otlp_exemplar **exemplars) +{ + size_t index; + otlp_exemplar *exemplar; + + buffer = cfl_sds_cat(buffer, " ex=[", 5); + for (index = 0; index < count; index++) { + exemplar = exemplars[index]; + buffer = sds_printf(buffer, "t%" PRIu64 ",", exemplar->time_unix_nano); + if (exemplar->value_case == OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_INT) { + buffer = sds_printf(buffer, "i%" PRId64, exemplar->as_int); + } + else { + buffer = cfl_sds_cat(buffer, "d", 1); + buffer = canon_double(buffer, exemplar->as_double); + } + buffer = cfl_sds_cat(buffer, ",tr", 3); + buffer = canon_bytes(buffer, exemplar->trace_id.data, exemplar->trace_id.len); + buffer = cfl_sds_cat(buffer, ",sp", 3); + buffer = canon_bytes(buffer, exemplar->span_id.data, exemplar->span_id.len); + buffer = canon_attributes(buffer, exemplar->n_filtered_attributes, + exemplar->filtered_attributes); + buffer = cfl_sds_cat(buffer, ";", 1); + } + return cfl_sds_cat(buffer, "]", 1); +} + +static cfl_sds_t canon_point_common(cfl_sds_t buffer, size_t attribute_count, + otlp_kv **attributes, uint64_t start, uint64_t time, + uint32_t flags) +{ + buffer = cfl_sds_cat(buffer, " attrs=", 7); + buffer = canon_attributes(buffer, attribute_count, attributes); + return sds_printf(buffer, " st=%" PRIu64 " t=%" PRIu64 " flags=%u", start, time, flags); +} + +static cfl_sds_t canon_optional_double(cfl_sds_t buffer, const char *name, + int present, double value) +{ + buffer = sds_printf(buffer, " %s=", name); + if (!present) { + return cfl_sds_cat(buffer, "-", 1); + } + return canon_double(buffer, value); +} + +static void canon_metric(struct lines *lines, cfl_sds_t prefix, otlp_metric *metric) +{ + size_t index; + size_t bucket; + cfl_sds_t line; + otlp_number_point **numbers = NULL; + size_t number_count = 0; + otlp_number_point *number; + otlp_histogram_point *histogram; + otlp_exp_point *exp; + otlp_summary_point *summary; + + prefix = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + prefix = sds_printf(prefix, "|metric name=%s desc=%s unit=%s meta=", + metric->name, metric->description, metric->unit); + prefix = canon_attributes(prefix, metric->n_metadata, metric->metadata); + + switch (metric->data_case) { + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE: + prefix = cfl_sds_cat(prefix, " gauge", 6); + numbers = metric->gauge->data_points; + number_count = metric->gauge->n_data_points; + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM: + prefix = sds_printf(prefix, " sum temporality=%d monotonic=%d", + metric->sum->aggregation_temporality, + metric->sum->is_monotonic ? 1 : 0); + numbers = metric->sum->data_points; + number_count = metric->sum->n_data_points; + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM: + prefix = sds_printf(prefix, " histogram temporality=%d", + metric->histogram->aggregation_temporality); + for (index = 0; index < metric->histogram->n_data_points; index++) { + histogram = metric->histogram->data_points[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, histogram->n_attributes, histogram->attributes, + histogram->start_time_unix_nano, + histogram->time_unix_nano, histogram->flags); + line = sds_printf(line, " count=%" PRIu64 " bounds=", histogram->count); + for (bucket = 0; bucket < histogram->n_explicit_bounds; bucket++) { + line = canon_double(line, histogram->explicit_bounds[bucket]); + line = cfl_sds_cat(line, ",", 1); + } + line = cfl_sds_cat(line, " buckets=", 9); + for (bucket = 0; bucket < histogram->n_bucket_counts; bucket++) { + line = sds_printf(line, "%" PRIu64 ",", histogram->bucket_counts[bucket]); + } + line = canon_optional_double(line, "sum", histogram->has_sum, histogram->sum); + line = canon_optional_double(line, "min", histogram->has_min, histogram->min); + line = canon_optional_double(line, "max", histogram->has_max, histogram->max); + line = canon_exemplars(line, histogram->n_exemplars, histogram->exemplars); + lines_add(lines, line); + } + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM: + prefix = sds_printf(prefix, " exp_histogram temporality=%d", + metric->exponential_histogram->aggregation_temporality); + for (index = 0; index < metric->exponential_histogram->n_data_points; index++) { + exp = metric->exponential_histogram->data_points[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, exp->n_attributes, exp->attributes, + exp->start_time_unix_nano, exp->time_unix_nano, + exp->flags); + line = sds_printf(line, " count=%" PRIu64 " scale=%d zero=%" PRIu64 " zt=", + exp->count, exp->scale, exp->zero_count); + line = canon_double(line, exp->zero_threshold); + /* Absent and empty bucket ranges carry the same information. */ + if (exp->positive != NULL && exp->positive->n_bucket_counts > 0) { + line = sds_printf(line, " pos@%d:", exp->positive->offset); + for (bucket = 0; bucket < exp->positive->n_bucket_counts; bucket++) { + line = sds_printf(line, "%" PRIu64 ",", exp->positive->bucket_counts[bucket]); + } + } + if (exp->negative != NULL && exp->negative->n_bucket_counts > 0) { + line = sds_printf(line, " neg@%d:", exp->negative->offset); + for (bucket = 0; bucket < exp->negative->n_bucket_counts; bucket++) { + line = sds_printf(line, "%" PRIu64 ",", exp->negative->bucket_counts[bucket]); + } + } + line = canon_optional_double(line, "sum", exp->has_sum, exp->sum); + line = canon_optional_double(line, "min", exp->has_min, exp->min); + line = canon_optional_double(line, "max", exp->has_max, exp->max); + line = canon_exemplars(line, exp->n_exemplars, exp->exemplars); + lines_add(lines, line); + } + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUMMARY: + prefix = cfl_sds_cat(prefix, " summary", 8); + for (index = 0; index < metric->summary->n_data_points; index++) { + summary = metric->summary->data_points[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, summary->n_attributes, summary->attributes, + summary->start_time_unix_nano, + summary->time_unix_nano, summary->flags); + line = sds_printf(line, " count=%" PRIu64 " sum=", summary->count); + line = canon_double(line, summary->sum); + line = cfl_sds_cat(line, " q=", 3); + for (bucket = 0; bucket < summary->n_quantile_values; bucket++) { + line = canon_double(line, summary->quantile_values[bucket]->quantile); + line = cfl_sds_cat(line, ":", 1); + line = canon_double(line, summary->quantile_values[bucket]->value); + line = cfl_sds_cat(line, ",", 1); + } + lines_add(lines, line); + } + break; + default: + prefix = cfl_sds_cat(prefix, " unset", 6); + break; + } + + for (index = 0; index < number_count; index++) { + number = numbers[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, number->n_attributes, number->attributes, + number->start_time_unix_nano, number->time_unix_nano, + number->flags); + if (number->value_case == OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT) { + line = sds_printf(line, " int=%" PRId64, number->as_int); + } + else { + line = cfl_sds_cat(line, " double=", 8); + line = canon_double(line, number->as_double); + } + line = canon_exemplars(line, number->n_exemplars, number->exemplars); + lines_add(lines, line); + } + cfl_sds_destroy(prefix); +} + +static void canon_request(struct lines *lines, otlp_request *value) +{ + size_t resource_index; + size_t scope_index; + size_t metric_index; + otlp_resource_metrics *resource_metrics; + otlp_scope_metrics *scope_metrics; + cfl_sds_t resource_prefix; + cfl_sds_t scope_prefix; + + for (resource_index = 0; resource_index < value->n_resource_metrics; resource_index++) { + resource_metrics = value->resource_metrics[resource_index]; + resource_prefix = cfl_sds_create("resource="); + if (!canon_include_context) { + resource_prefix = cfl_sds_cat(resource_prefix, "*", 1); + } + else if (resource_metrics->resource != NULL) { + resource_prefix = canon_attributes(resource_prefix, + resource_metrics->resource->n_attributes, + resource_metrics->resource->attributes); + resource_prefix = sds_printf(resource_prefix, " dropped=%u", + resource_metrics->resource->dropped_attributes_count); + } + resource_prefix = sds_printf(resource_prefix, " schema=%s", + !canon_include_context ? "*" : + resource_metrics->schema_url ? + resource_metrics->schema_url : ""); + for (scope_index = 0; scope_index < resource_metrics->n_scope_metrics; scope_index++) { + scope_metrics = resource_metrics->scope_metrics[scope_index]; + scope_prefix = cfl_sds_create_len(resource_prefix, cfl_sds_len(resource_prefix)); + scope_prefix = cfl_sds_cat(scope_prefix, "|scope=", 7); + if (!canon_include_context) { + scope_prefix = cfl_sds_cat(scope_prefix, "*", 1); + } + else if (scope_metrics->scope != NULL) { + scope_prefix = sds_printf(scope_prefix, "%s@%s dropped=%u ", + scope_metrics->scope->name, + scope_metrics->scope->version, + scope_metrics->scope->dropped_attributes_count); + scope_prefix = canon_attributes(scope_prefix, + scope_metrics->scope->n_attributes, + scope_metrics->scope->attributes); + } + scope_prefix = sds_printf(scope_prefix, " schema=%s", + !canon_include_context ? "*" : + scope_metrics->schema_url ? scope_metrics->schema_url : ""); + for (metric_index = 0; metric_index < scope_metrics->n_metrics; metric_index++) { + canon_metric(lines, scope_prefix, scope_metrics->metrics[metric_index]); + } + cfl_sds_destroy(scope_prefix); + } + cfl_sds_destroy(resource_prefix); + } +} + +/* Pipelines */ + +static struct cmt *msgpack_hop(struct cmt *context, int destroy_source) +{ + struct cmt *decoded = NULL; + char *buffer; + size_t size; + size_t offset = 0; + int result; + + result = cmt_encode_msgpack_create(context, &buffer, &size); + TEST_CHECK(result == 0); + if (result != 0) { + return NULL; + } + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(offset == size); + cmt_encode_msgpack_destroy(buffer); + if (destroy_source) { + cmt_decode_msgpack_destroy(context); + } + return result == CMT_DECODE_MSGPACK_SUCCESS ? decoded : NULL; +} + +/* Run one pipeline on a decoded context and append the OTLP output lines. */ +static int run_pipeline(struct lines *lines, struct cmt *context, int pipeline) +{ + struct cmt *current; + struct cmt *merged; + cfl_sds_t wire; + otlp_request *output; + + current = context; + merged = NULL; + if (pipeline != PIPELINE_DIRECT) { + current = msgpack_hop(context, CMT_FALSE); + if (current == NULL) { + return -1; + } + } + if (pipeline == PIPELINE_FORWARD) { + current = msgpack_hop(current, CMT_TRUE); + if (current == NULL) { + return -1; + } + } + if (pipeline == PIPELINE_CAT) { + merged = cmt_create(); + TEST_ASSERT(merged != NULL); + TEST_CHECK(cmt_cat(merged, current) == 0); + cmt_decode_msgpack_destroy(current); + current = merged; + } + + wire = cmt_encode_opentelemetry_create(current); + TEST_CHECK(wire != NULL); + if (wire != NULL) { + output = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_CHECK(output != NULL); + if (output != NULL) { + canon_request(lines, output); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked( + output, NULL); + } + cmt_encode_opentelemetry_destroy(wire); + } + + if (merged != NULL) { + cmt_destroy(merged); + } + else if (current != context) { + cmt_decode_msgpack_destroy(current); + } + return wire != NULL ? 0 : -1; +} + +static cfl_sds_t pack_request(otlp_request *value) +{ + size_t size; + uint8_t *buffer; + cfl_sds_t payload; + + size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(value); + buffer = require(calloc(1, size + 1)); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(value, buffer); + payload = cfl_sds_create_len((char *) buffer, size); + free(buffer); + TEST_ASSERT(payload != NULL); + return payload; +} + +static void compare_lines(struct lines *expected, struct lines *actual, const char *pipeline) +{ + size_t index; + int equal; + + if (expected->count > 1) { + qsort(expected->entries, expected->count, sizeof(cfl_sds_t), compare_strings); + } + if (actual->count > 1) { + qsort(actual->entries, actual->count, sizeof(cfl_sds_t), compare_strings); + } + equal = expected->count == actual->count; + for (index = 0; equal && index < expected->count; index++) { + equal = strcmp(expected->entries[index], actual->entries[index]) == 0; + } + TEST_CHECK_(equal, "pipeline %s reproduces every data point", pipeline); + if (!equal) { + for (index = 0; index < expected->count; index++) { + TEST_MSG("expected: %s", expected->entries[index]); + } + for (index = 0; index < actual->count; index++) { + TEST_MSG("actual: %s", actual->entries[index]); + } + } +} + +/* + * Decode the request as in_opentelemetry does and check that every requested + * pipeline reproduces it. The canonical lines of 'expected' default to the + * input itself; a case passes its own expectation when a transformation is + * intentionally lossy. + */ +static void check_pipelines(otlp_request *input, otlp_request *expected, unsigned int pipelines) +{ + struct cfl_list contexts; + struct cfl_list *head; + struct lines expected_lines; + struct lines actual_lines; + struct cmt *context; + cfl_sds_t payload; + size_t offset; + int pipeline; + int result; + + payload = pack_request(input); + + for (pipeline = 0; pipeline < PIPELINE_COUNT; pipeline++) { + if ((pipelines & (1u << pipeline)) == 0) { + continue; + } + offset = 0; + result = cmt_decode_opentelemetry_create(&contexts, payload, cfl_sds_len(payload), + &offset); + TEST_CHECK_(result == CMT_DECODE_OPENTELEMETRY_SUCCESS, + "OTLP decode (result %d)", result); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + break; + } + canon_include_context = pipeline != PIPELINE_CAT; + memset(&expected_lines, 0, sizeof(expected_lines)); + memset(&actual_lines, 0, sizeof(actual_lines)); + canon_request(&expected_lines, expected != NULL ? expected : input); + cfl_list_foreach(head, &contexts) { + context = cfl_list_entry(head, struct cmt, _head); + TEST_CHECK(run_pipeline(&actual_lines, context, pipeline) == 0); + } + compare_lines(&expected_lines, &actual_lines, pipeline_names[pipeline]); + canon_include_context = CMT_TRUE; + lines_destroy(&expected_lines); + lines_destroy(&actual_lines); + cmt_decode_opentelemetry_destroy(&contexts); + } + + cfl_sds_destroy(payload); +} + +#define ALL_PIPELINES ((1u << PIPELINE_COUNT) - 1) + +/* Cases */ + +#define RESOURCE_ATTRIBUTES \ + 2, kv_list(2, kv("service.name", any_string("checkout")), \ + kv("host.name", any_string("node-1"))) + +static otlp_request *single_metric(otlp_metric *metric) +{ + return request(1, resource(RESOURCE_ATTRIBUTES, 1, + scope("io.opentelemetry.runtime", "1.2.0", 0, NULL, 1, metric))); +} + +static otlp_kv **attrs1(const char *key, otlp_any *value) +{ + return kv_list(1, kv(key, value)); +} + +static double double_from_bits(uint64_t bits) +{ + double value; + + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static void run_case(otlp_request *input) +{ + check_pipelines(input, NULL, ALL_PIPELINES); + test_alloc_release(); +} + +/* Counter: monotonic cumulative sum with integer and double points. */ +static void test_sum_monotonic_cumulative(void) +{ + run_case(single_metric( + metric_sum("http.server.requests", "requests", "{request}", TEMPORALITY_CUMULATIVE, 1, 2, + number_int(1000, 2000, 42, 1, attrs1("http.route", any_string("/a"))), + number_int(1000, 2000, 7, 1, attrs1("http.route", any_string("/b")))))); +} + +/* UpDownCounter: non-monotonic sums must not become monotonic counters. */ +static void test_sum_non_monotonic(void) +{ + run_case(single_metric( + metric_sum("queue.items", "items", "{item}", TEMPORALITY_CUMULATIVE, 0, 2, + number_int(1000, 2000, -5, 1, attrs1("queue", any_string("in"))), + number_double(1000, 2000, -0.25, 1, attrs1("queue", any_string("out")))))); +} + +/* SDKs configured for delta temporality (common with some backends). */ +static void test_sum_delta(void) +{ + run_case(single_metric( + metric_sum("bytes.sent", "bytes", "By", TEMPORALITY_DELTA, 1, 1, + number_int(1000, 2000, 512, 1, attrs1("peer", any_string("db")))))); +} + +static void test_gauge_integer_extremes(void) +{ + run_case(single_metric( + metric_gauge("extremes", "integer extremes", "1", 4, + number_int(0, 100, INT64_MIN, 1, attrs1("v", any_string("min"))), + number_int(0, 100, INT64_MAX, 1, attrs1("v", any_string("max"))), + number_int(0, 100, -1, 1, attrs1("v", any_string("minus_one"))), + number_int(0, 100, 9007199254740993LL, 1, attrs1("v", any_string("2^53+1")))))); +} + +/* Prometheus receivers emit staleness markers (0x7ff0000000000002) and + * applications report NaN, infinities and negative zero. */ +static void test_gauge_special_doubles(void) +{ + run_case(single_metric( + metric_gauge("specials", "special doubles", "1", 6, + number_double(0, 100, double_from_bits(0x7ff8000000000000ULL), 1, + attrs1("v", any_string("nan"))), + number_double(0, 100, double_from_bits(0x7ff0000000000002ULL), 1, + attrs1("v", any_string("stale"))), + number_double(0, 100, INFINITY, 1, attrs1("v", any_string("+inf"))), + number_double(0, 100, -INFINITY, 1, attrs1("v", any_string("-inf"))), + number_double(0, 100, -0.0, 1, attrs1("v", any_string("-0"))), + number_double(0, 100, 4.9406564584124654e-324, 1, attrs1("v", any_string("denormal")))))); +} + +static void test_point_flags_and_timestamps(void) +{ + otlp_number_point *missing; + + missing = number_double(10, 20, 0, 1, attrs1("v", any_string("no_value"))); + missing->flags = 1; + run_case(single_metric( + metric_gauge("timestamps", "flags and timestamps", "1", 3, + missing, + number_double(0, 9223372036854775813ULL, 1, 1, attrs1("v", any_string("late"))), + number_double(500, 400, 2, 1, attrs1("v", any_string("start_after_time")))))); +} + +static void test_histogram_explicit_bounds(void) +{ + static const double bounds[] = {0.005, 0.01, 0.1, 1, 10}; + static const uint64_t counts_a[] = {1, 2, 3, 4, 5, 6}; + static const uint64_t counts_b[] = {0, 0, 0, 0, 0, 9}; + otlp_histogram_point *a; + otlp_histogram_point *b; + + a = histogram_point(1000, 2000, 5, bounds, counts_a, 1, + attrs1("http.route", any_string("/a"))); + a->has_sum = 1; + a->sum = 123.5; + a->has_min = 1; + a->min = 0.001; + a->has_max = 1; + a->max = 42; + a->n_exemplars = 2; + a->exemplars = exemplar_list(2, + exemplar_double(1500, 0.007, "0123456789abcdef", "01234567", 1, + attrs1("user", any_string("u1"))), + exemplar_int(1600, 3)); + b = histogram_point(1000, 2000, 5, bounds, counts_b, 1, + attrs1("http.route", any_string("/b"))); + b->has_sum = 1; + b->sum = 900; + run_case(single_metric(metric_histogram("http.server.duration", "s", + TEMPORALITY_CUMULATIVE, 2, a, b))); +} + +static void test_histogram_delta_without_sum(void) +{ + static const double bounds[] = {1, 2}; + static const uint64_t counts[] = {3, 0, 1}; + + run_case(single_metric(metric_histogram("latency.delta", "ms", TEMPORALITY_DELTA, 1, + histogram_point(1000, 2000, 2, bounds, counts, 1, + attrs1("op", any_string("get")))))); +} + +/* + * A histogram with no explicit bounds has a single +Inf bucket (SDK views use + * it to keep only count and sum). CMetrics cannot represent it: the request is + * rejected. Accepting it would also emit a zero-bucket MessagePack histogram + * that every released reader rejects, so this is pinned until decided. + */ +static void test_histogram_without_bounds_is_rejected(void) +{ + static const uint64_t counts[] = {17}; + struct cfl_list contexts; + cfl_sds_t payload; + size_t offset = 0; + int result; + + payload = pack_request(single_metric(metric_histogram("single.bucket", "s", + TEMPORALITY_CUMULATIVE, 1, histogram_point(1000, 2000, 0, NULL, counts, 0, NULL)))); + result = cmt_decode_opentelemetry_create(&contexts, payload, cfl_sds_len(payload), &offset); + TEST_CHECK(result != CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cmt_decode_opentelemetry_destroy(&contexts); + } + cfl_sds_destroy(payload); + test_alloc_release(); +} + +static void test_exponential_histogram_layouts(void) +{ + static const uint64_t positive[] = {1, 0, 4, 2}; + static const uint64_t negative[] = {3, 1}; + static const uint64_t fine[] = {5}; + otlp_exp_point *coarse; + otlp_exp_point *precise; + otlp_exp_point *zeros; + + coarse = exp_point(1000, 2000, -3, 2, 1e-9, exp_buckets(-5, 4, positive), + exp_buckets(2, 2, negative), 1, attrs1("layout", any_string("coarse"))); + coarse->has_sum = 1; + coarse->sum = -12.5; + coarse->has_min = 1; + coarse->min = -40; + coarse->has_max = 1; + coarse->max = 300; + precise = exp_point(1000, 2000, 20, 0, 0, exp_buckets(1048576, 1, fine), NULL, 1, + attrs1("layout", any_string("precise"))); + precise->has_sum = 1; + precise->sum = 10; + zeros = exp_point(1000, 2000, 0, 6, 0.5, NULL, NULL, 1, + attrs1("layout", any_string("zeros"))); + run_case(single_metric(metric_exp_histogram("rpc.duration", TEMPORALITY_DELTA, 3, + coarse, precise, zeros))); +} + +static void test_summary_quantiles(void) +{ + static const double quantiles[] = {0, 0.5, 0.99, 1}; + static const double values[] = {0.1, 2.5, 9.75, 12}; + + run_case(single_metric(metric_summary("gc.pause", 2, + summary_point(1000, 2000, 40, 100.5, 4, quantiles, values, 1, + attrs1("gen", any_string("young"))), + summary_point(1000, 2000, 2, 30, 4, quantiles, values, 1, + attrs1("gen", any_string("old")))))); +} + +static void test_summary_without_quantiles(void) +{ + run_case(single_metric(metric_summary("requests.summary", 1, + summary_point(1000, 2000, 5, 12.5, 0, NULL, NULL, 1, + attrs1("route", any_string("/")))))); +} + +/* Attribute values of every OTLP type keep their type end to end. */ +static void test_typed_attributes(void) +{ + run_case(single_metric(metric_gauge("typed", "typed attributes", "1", 1, + number_double(0, 100, 1, 9, kv_list(9, + kv("string", any_string("value")), + kv("empty", any_string("")), + kv("unicode", any_string("caf\xc3\xa9 \xe2\x82\xac")), + kv("int", any_int(-42)), + kv("double", any_double(2.5)), + kv("bool", any_bool(1)), + kv("bytes", any_bytes("\x00\xff\x10", 3)), + kv("array", any_array(3, any_string("a"), any_int(1), any_bool(0))), + kv("map", any_kvlist(2, kv("inner", any_string("x")), kv("n", any_double(0.5))))))))); +} + +static otlp_request *type_identity_request(int collapsed) +{ + if (collapsed) { + return single_metric(metric_gauge("identity", "type identity", "1", 1, + number_double(0, 100, 3, 1, attrs1("k", any_double(7))))); + } + return single_metric(metric_gauge("identity", "type identity", "1", 3, + number_double(0, 100, 1, 1, attrs1("k", any_int(7))), + number_double(0, 100, 2, 1, attrs1("k", any_string("7"))), + number_double(0, 100, 3, 1, attrs1("k", any_double(7))))); +} + +/* + * int 7, string "7" and double 7 are three OTLP series and stay three through + * OTLP and MessagePack. cmt_cat matches series by their string labels, which + * are identical, so it merges them and the last point wins. Pinned: series + * identity in cmt_cat does not consider attribute types. + */ +static void test_attribute_type_identity(void) +{ + check_pipelines(type_identity_request(CMT_FALSE), NULL, + ALL_PIPELINES & ~(1u << PIPELINE_CAT)); + check_pipelines(type_identity_request(CMT_FALSE), type_identity_request(CMT_TRUE), + 1u << PIPELINE_CAT); + test_alloc_release(); +} + +/* Native metadata as a released relay carries it: opaque context metadata + * written back unchanged, whatever it does to the points. */ +static struct cfl_kvlist *relay_copy(struct cmt *context, int internal) +{ + struct cfl_kvlist *copy = NULL; + mpack_writer_t writer; + mpack_reader_t reader; + char *buffer = NULL; + size_t size = 0; + int result; + + mpack_writer_init_growable(&writer, &buffer, &size); + result = internal ? cmt_otlp_metadata_pack(&writer, context) : + cmt_otlp_metadata_pack_external(&writer, context); + TEST_ASSERT(mpack_writer_destroy(&writer) == mpack_ok && result == 0); + mpack_reader_init_data(&reader, buffer, size); + result = unpack_cfl_kvlist(&reader, ©); + TEST_ASSERT(mpack_reader_destroy(&reader) == mpack_ok && result == 0); + free(buffer); + return copy; +} + +/* Distribution points that differ only by attribute type, count and sum, + * but not by distribution, stay distinct through every hop. */ +static void test_distribution_type_identity(void) +{ + static const double bounds[] = {1, 2}; + static const uint64_t buckets_a[] = {1, 0, 1}; + static const uint64_t buckets_b[] = {0, 2, 0}; + static const double quantiles[] = {0.5}; + static const double quantile_a[] = {1}; + static const double quantile_b[] = {2}; + static const uint64_t positive_a[] = {1, 1}; + static const uint64_t positive_b[] = {2, 0}; + unsigned int mask = ALL_PIPELINES & ~(1u << PIPELINE_CAT); + otlp_histogram_point *histogram[2]; + otlp_summary_point *summary[2]; + otlp_exp_point *exp[2]; + int index; + + histogram[0] = histogram_point(0, 100, 2, bounds, buckets_a, 1, attrs1("k", any_int(7))); + histogram[1] = histogram_point(0, 100, 2, bounds, buckets_b, 1, attrs1("k", any_string("7"))); + summary[0] = summary_point(0, 100, 2, 3, 1, quantiles, quantile_a, 1, attrs1("k", any_int(7))); + summary[1] = summary_point(0, 100, 2, 3, 1, quantiles, quantile_b, 1, + attrs1("k", any_string("7"))); + exp[0] = exp_point(0, 100, 0, 0, 0, exp_buckets(0, 2, positive_a), NULL, 1, + attrs1("k", any_int(7))); + exp[1] = exp_point(0, 100, 0, 0, 0, exp_buckets(0, 2, positive_b), NULL, 1, + attrs1("k", any_string("7"))); + for (index = 0; index < 2; index++) { + histogram[index]->has_sum = 1; + histogram[index]->sum = 3; + exp[index]->has_sum = 1; + exp[index]->sum = 3; + } + histogram[0]->flags = 1; + summary[0]->flags = 1; + exp[0]->flags = 1; + + check_pipelines(single_metric(metric_histogram("latency", "s", TEMPORALITY_CUMULATIVE, 2, + histogram[0], histogram[1])), NULL, mask); + check_pipelines(single_metric(metric_summary("rpc", 2, summary[0], summary[1])), NULL, mask); + check_pipelines(single_metric(metric_exp_histogram("exp", TEMPORALITY_CUMULATIVE, 2, + exp[0], exp[1])), NULL, mask); + test_alloc_release(); +} + +struct typed_relay { + struct cfl_list contexts; + struct cmt *decoded; + char *buffer; + cfl_sds_t payload; + cfl_sds_t wire; + otlp_request *output; +}; + +/* Points of one series and time that differ only by attribute type: integer + * 7, string "7" and double 7.0. */ +static otlp_request *typed_points(const double *values, const uint32_t *flags) +{ + otlp_request *request; + otlp_metric *metric; + size_t index; + + request = single_metric(metric_gauge("identity", "type identity", "1", 3, + number_double(0, 100, values[0], 1, attrs1("k", any_int(7))), + number_double(0, 100, values[1], 1, attrs1("k", any_string("7"))), + number_double(0, 100, values[2], 1, attrs1("k", any_double(7))))); + metric = request->resource_metrics[0]->scope_metrics[0]->metrics[0]; + for (index = 0; index < 3; index++) { + metric->gauge->data_points[index]->flags = flags[index]; + } + return request; +} + +/* Decode the request, then act as a released relay: keep native metadata as + * opaque context metadata and either drop the point at drop_index or swap + * the first two points, then decode the relayed chunk and export it. */ +static void typed_relay_run(struct typed_relay *relay, otlp_request *input, + int drop_index, int swap) +{ + struct cfl_list *head; + struct cfl_list *first; + struct cmt *context; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + size_t size; + size_t offset = 0; + int index = 0; + + memset(relay, 0, sizeof(*relay)); + relay->payload = pack_request(input); + TEST_ASSERT(cmt_decode_opentelemetry_create(&relay->contexts, relay->payload, + cfl_sds_len(relay->payload), &offset) == + CMT_DECODE_OPENTELEMETRY_SUCCESS); + context = cfl_list_entry_first(&relay->contexts, struct cmt, _head); + gauge = cfl_list_entry_first(&context->gauges, struct cmt_gauge, _head); + TEST_ASSERT(cfl_list_size(&gauge->map->metrics) == 3); + + internal = relay_copy(context, CMT_TRUE); + external = relay_copy(context, CMT_FALSE); + cfl_kvlist_destroy(gauge->map->otlp_metadata); + gauge->map->otlp_metadata = NULL; + cfl_list_foreach(head, &gauge->map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = NULL; + } + cfl_kvlist_destroy(context->internal_metadata); + context->internal_metadata = internal; + cfl_kvlist_destroy(context->external_metadata); + context->external_metadata = external; + + if (swap) { + first = gauge->map->metrics.next; + cfl_list_del(first); + cfl_list_add_after(first, gauge->map->metrics.next, &gauge->map->metrics); + } + cfl_list_foreach(head, &gauge->map->metrics) { + if (index++ == drop_index) { + cmt_map_metric_destroy(cfl_list_entry(head, struct cmt_metric, _head)); + break; + } + } + + TEST_ASSERT(cmt_encode_msgpack_create(context, &relay->buffer, &size) == 0); + offset = 0; + TEST_ASSERT(cmt_decode_msgpack_create(&relay->decoded, relay->buffer, size, &offset) == 0); + relay->wire = cmt_encode_opentelemetry_create(relay->decoded); + TEST_ASSERT(relay->wire != NULL); + relay->output = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(relay->wire), (const uint8_t *) relay->wire); + TEST_ASSERT(relay->output != NULL); +} + +static void typed_relay_destroy(struct typed_relay *relay) +{ + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked( + relay->output, NULL); + cmt_encode_opentelemetry_destroy(relay->wire); + cmt_decode_msgpack_destroy(relay->decoded); + cmt_encode_msgpack_destroy(relay->buffer); + cmt_decode_opentelemetry_destroy(&relay->contexts); + cfl_sds_destroy(relay->payload); +} + +/* Each exported point carries the flags and attribute type recorded for its + * value, or, when values cannot tell points apart, neither. */ +static void check_typed_points(struct typed_relay *relay, size_t expected_points, + const double *values, const uint32_t *flags, + int distinguishable) +{ + static const int cases[] = { + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE, + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE, + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE + }; + otlp_metric *metric; + otlp_number_point *point; + size_t index; + size_t source; + + metric = relay->output->resource_metrics[0]->scope_metrics[0]->metrics[0]; + TEST_CHECK(metric->gauge->n_data_points == expected_points); + for (index = 0; index < metric->gauge->n_data_points; index++) { + point = metric->gauge->data_points[index]; + TEST_ASSERT(point->n_attributes == 1); + if (!distinguishable) { + TEST_CHECK_(point->flags == 0, "point %g carries no recorded flags", + point->as_double); + TEST_CHECK_(point->attributes[0]->value->value_case == cases[1], + "point %g keeps its label-derived attribute", point->as_double); + continue; + } + for (source = 0; source < 3 && values[source] != point->as_double; source++); + TEST_ASSERT(source < 3); + TEST_CHECK_(point->flags == flags[source], "point %g keeps its own flags", + point->as_double); + TEST_CHECK_(point->attributes[0]->value->value_case == cases[source], + "point %g keeps its own attribute type", point->as_double); + } +} + +/* Points that differ only by attribute type share labels; their values tell + * them apart through any relay edit. Points identical in value too cannot be + * told apart, so none of them may take metadata recorded for another, from + * the native sidecar or from the legacy tree. */ +static void test_attribute_type_identity_after_relay(void) +{ + static const double distinct[] = {1, 2, 3}; + static const double equal[] = {5, 5, 5}; + static const uint32_t first_flagged[] = {1, 0, 0}; + static const uint32_t last_flagged[] = {0, 0, 1}; + struct typed_relay relay; + + /* drop the first point */ + typed_relay_run(&relay, typed_points(distinct, first_flagged), 0, CMT_FALSE); + check_typed_points(&relay, 2, distinct, first_flagged, CMT_TRUE); + typed_relay_destroy(&relay); + + /* swap the first two points: count and identities are unchanged */ + typed_relay_run(&relay, typed_points(distinct, first_flagged), -1, CMT_TRUE); + check_typed_points(&relay, 3, distinct, first_flagged, CMT_TRUE); + typed_relay_destroy(&relay); + typed_relay_run(&relay, typed_points(equal, first_flagged), -1, CMT_TRUE); + check_typed_points(&relay, 3, equal, first_flagged, CMT_FALSE); + typed_relay_destroy(&relay); + + /* drop the flagged last point: its legacy entry shares the survivors' + * series hash and time */ + typed_relay_run(&relay, typed_points(distinct, last_flagged), 2, CMT_FALSE); + check_typed_points(&relay, 2, distinct, last_flagged, CMT_TRUE); + typed_relay_destroy(&relay); + typed_relay_run(&relay, typed_points(equal, last_flagged), 2, CMT_FALSE); + check_typed_points(&relay, 2, equal, last_flagged, CMT_FALSE); + typed_relay_destroy(&relay); + + test_alloc_release(); +} + +/* SDKs do not order attributes; series with the same keys in a different + * order must keep their own values. */ +static void test_attribute_order(void) +{ + run_case(single_metric(metric_gauge("order", "attribute order", "1", 2, + number_double(0, 100, 1, 2, kv_list(2, kv("a", any_string("1")), kv("b", any_string("2")))), + number_double(0, 100, 2, 2, kv_list(2, kv("b", any_string("3")), kv("a", any_string("4"))))))); +} + +/* Points of one metric with different attribute key sets. */ +static void test_attribute_key_sets(void) +{ + run_case(single_metric(metric_gauge("key_sets", "key sets", "1", 3, + number_double(0, 100, 1, 0, NULL), + number_double(0, 100, 2, 1, attrs1("a", any_string("x"))), + number_double(0, 100, 3, 2, kv_list(2, kv("a", any_string("x")), + kv("b", any_string("y"))))))); +} + +/* A collector forwarding several services in one request. */ +static void test_multiple_resources_and_scopes(void) +{ + run_case(request(2, + resource(1, attrs1("service.name", any_string("cart")), 2, + scope("lib.http", "1.0", 0, NULL, 1, + metric_gauge("shared.name", "from http", "1", 1, + number_double(0, 100, 1, 0, NULL))), + scope("lib.db", "2.0", 1, attrs1("db.system", any_string("postgresql")), 1, + metric_sum("shared.name", "from db", "1", TEMPORALITY_CUMULATIVE, 1, 1, + number_int(0, 100, 5, 0, NULL)))), + resource(1, attrs1("service.name", any_string("payment")), 1, + scope("lib.http", "1.0", 0, NULL, 1, + metric_gauge("shared.name", "from http", "1", 1, + number_double(0, 100, 2, 0, NULL)))))); +} + +static otlp_request *details_request(const char *description) +{ + otlp_request *input; + + input = single_metric(metric_gauge("details", description, "", 1, + number_double(0, 100, 1, 0, NULL))); + input->resource_metrics[0]->schema_url = "https://opentelemetry.io/schemas/1.24.0"; + input->resource_metrics[0]->resource->dropped_attributes_count = 3; + input->resource_metrics[0]->scope_metrics[0]->schema_url = + "https://opentelemetry.io/schemas/1.21.0"; + input->resource_metrics[0]->scope_metrics[0]->scope->dropped_attributes_count = 1; + input->resource_metrics[0]->scope_metrics[0]->scope->n_attributes = 1; + input->resource_metrics[0]->scope_metrics[0]->scope->attributes = + attrs1("scope.kind", any_string("auto")); + return input; +} + +/* Schema URLs, dropped counts and scope attributes. CMetrics families require + * a description, so an empty one is emitted as "-" (pinned). */ +static void test_resource_and_scope_details(void) +{ + check_pipelines(details_request(""), details_request("-"), ALL_PIPELINES); + test_alloc_release(); +} + +static void test_metric_metadata(void) +{ + otlp_metric *metric; + + metric = metric_gauge("with.metadata", "has metadata", "By", 1, + number_double(0, 100, 1, 0, NULL)); + metric->n_metadata = 2; + metric->metadata = kv_list(2, kv("prometheus.type", any_string("gauge")), + kv("weight", any_int(3))); + run_case(single_metric(metric)); +} + +static void test_number_exemplars(void) +{ + otlp_number_point *point; + + point = number_int(1000, 2000, 10, 1, attrs1("route", any_string("/"))); + point->n_exemplars = 2; + point->exemplars = exemplar_list(2, + exemplar_int(1500, 4), + exemplar_double(1600, 6.5, "fedcba9876543210", "76543210", 2, + kv_list(2, kv("user", any_string("u2")), kv("n", any_int(1))))); + run_case(single_metric(metric_sum("exemplars", "with exemplars", "1", + TEMPORALITY_CUMULATIVE, 1, 1, point))); +} + +/* Large cardinality as produced by busy collectors. */ +static void test_many_series(void) +{ + otlp_metric *metric; + char *values; + size_t index; + + metric = metric_gauge("many.series", "cardinality", "1", 0); + metric->gauge->n_data_points = 2000; + metric->gauge->data_points = test_alloc(sizeof(otlp_number_point *) * 2000); + values = test_alloc(2000 * 8); + for (index = 0; index < 2000; index++) { + snprintf(&values[index * 8], 8, "%zu", index); + metric->gauge->data_points[index] = + number_double(0, 100, (double) index, 1, + attrs1("series", any_string(&values[index * 8]))); + } + run_case(single_metric(metric)); +} + +/* Every prefix and single byte corruption of a real request must be handled + * without crashing; whatever decodes must also survive a MessagePack hop. */ +static void test_truncated_and_corrupted_requests(void) +{ + static const double bounds[] = {1, 2}; + static const uint64_t counts[] = {1, 2, 3}; + struct cfl_list contexts; + struct cfl_list *head; + struct cmt *context; + struct cmt *decoded; + cfl_sds_t payload; + cfl_sds_t copy; + size_t length; + size_t offset; + int result; + + payload = pack_request(request(1, resource(RESOURCE_ATTRIBUTES, 1, + scope("s", "1", 1, attrs1("k", any_string("v")), 3, + metric_sum("c", "d", "1", TEMPORALITY_CUMULATIVE, 1, 1, + number_int(1, 2, 3, 1, attrs1("a", any_int(1)))), + metric_histogram("h", "s", TEMPORALITY_CUMULATIVE, 1, + histogram_point(1, 2, 2, bounds, counts, 1, attrs1("a", any_bool(1)))), + metric_exp_histogram("e", TEMPORALITY_DELTA, 1, + exp_point(1, 2, 1, 1, 0, exp_buckets(-1, 3, counts), NULL, 0, NULL)))))); + + for (length = 0; length <= cfl_sds_len(payload) * 2; length++) { + copy = cfl_sds_create_len(payload, length <= cfl_sds_len(payload) ? + length : cfl_sds_len(payload)); + if (length > cfl_sds_len(payload)) { + copy[length - cfl_sds_len(payload) - 1] ^= 0x5a; + } + offset = 0; + result = cmt_decode_opentelemetry_create(&contexts, copy, cfl_sds_len(copy), &offset); + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cfl_list_foreach(head, &contexts) { + context = cfl_list_entry(head, struct cmt, _head); + decoded = msgpack_hop(context, CMT_FALSE); + if (decoded != NULL) { + cmt_decode_msgpack_destroy(decoded); + } + } + cmt_decode_opentelemetry_destroy(&contexts); + } + cfl_sds_destroy(copy); + } + cfl_sds_destroy(payload); + test_alloc_release(); +} + +/* Containers without data must not produce data points or fail. */ +static void test_empty_containers(void) +{ + otlp_request *input; + + input = request(3, + resource(RESOURCE_ATTRIBUTES, 0), + resource(RESOURCE_ATTRIBUTES, 1, scope("empty", "1", 0, NULL, 0)), + resource(RESOURCE_ATTRIBUTES, 1, scope("s", "1", 0, NULL, 1, + metric_gauge("no.points", "", "", 0)))); + check_pipelines(input, request(0), ALL_PIPELINES); + test_alloc_release(); +} + +TEST_LIST = { + {"sum_monotonic_cumulative", test_sum_monotonic_cumulative}, + {"sum_non_monotonic", test_sum_non_monotonic}, + {"sum_delta", test_sum_delta}, + {"gauge_integer_extremes", test_gauge_integer_extremes}, + {"gauge_special_doubles", test_gauge_special_doubles}, + {"point_flags_and_timestamps", test_point_flags_and_timestamps}, + {"histogram_explicit_bounds", test_histogram_explicit_bounds}, + {"histogram_delta_without_sum", test_histogram_delta_without_sum}, + {"histogram_without_bounds_is_rejected", test_histogram_without_bounds_is_rejected}, + {"exponential_histogram_layouts", test_exponential_histogram_layouts}, + {"summary_quantiles", test_summary_quantiles}, + {"summary_without_quantiles", test_summary_without_quantiles}, + {"typed_attributes", test_typed_attributes}, + {"attribute_type_identity", test_attribute_type_identity}, + {"attribute_type_identity_after_relay", test_attribute_type_identity_after_relay}, + {"distribution_type_identity", test_distribution_type_identity}, + {"attribute_order", test_attribute_order}, + {"attribute_key_sets", test_attribute_key_sets}, + {"multiple_resources_and_scopes", test_multiple_resources_and_scopes}, + {"resource_and_scope_details", test_resource_and_scope_details}, + {"metric_metadata", test_metric_metadata}, + {"number_exemplars", test_number_exemplars}, + {"many_series", test_many_series}, + {"truncated_and_corrupted_requests", test_truncated_and_corrupted_requests}, + {"empty_containers", test_empty_containers}, + {0} +}; From ec49ff81dd6cfae82b681eb075255e3831dce09a Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Thu, 22 Oct 2026 17:13:11 -0600 Subject: [PATCH 7/7] tests: decode MessagePack written by released Fluent Bit versions Add fixtures written by the CMetrics bundled with Fluent Bit 5.0.0, 5.0.11 and 5.1.5 and check their content after decode, re-encode, cmt_cat() and mixed forward batches. Also check that the encoder only writes keys released readers accept, scalar extremes, every truncation and byte flip of a chunk, oversized declared counts and the strict format version. Signed-off-by: Eduardo Silva --- tests/CMakeLists.txt | 1 + tests/data/all-types-release.c | 103 ++++ tests/data/all-types-v5.0.0.msgpack | Bin 0 -> 1738 bytes tests/data/all-types-v5.0.11.msgpack | Bin 0 -> 1793 bytes tests/data/all-types-v5.1.5.msgpack | Bin 0 -> 1829 bytes tests/msgpack_compat.c | 826 +++++++++++++++++++++++++++ 6 files changed, 930 insertions(+) create mode 100644 tests/data/all-types-release.c create mode 100644 tests/data/all-types-v5.0.0.msgpack create mode 100644 tests/data/all-types-v5.0.11.msgpack create mode 100644 tests/data/all-types-v5.1.5.msgpack create mode 100644 tests/msgpack_compat.c diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 39e031c0..06555709 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,6 +21,7 @@ set(UNIT_TESTS_FILES format_conversion.c expire.c otlp_pipeline.c + msgpack_compat.c ) if (CMT_BUILD_PROMETHEUS_TEXT_DECODER) diff --git a/tests/data/all-types-release.c b/tests/data/all-types-release.c new file mode 100644 index 00000000..fbf53d38 --- /dev/null +++ b/tests/data/all-types-release.c @@ -0,0 +1,103 @@ +/* + * Generator for all-types-v5.0.0.msgpack, all-types-v5.0.11.msgpack and + * all-types-v5.1.5.msgpack: one context with every metric type, written by the + * CMetrics library bundled with that Fluent Bit release (2.0.5, 2.1.5 and + * 2.2.5). These are the payloads those agents send over forward and keep in + * filesystem chunks, so current readers must keep decoding them. + * + * Build it against the headers and static libraries of lib/cmetrics, lib/cfl + * and lib/fluent-otel-proto from the Fluent Bit release tag and redirect stdout + * to the fixture. The fixtures are versioned on purpose: tests must never + * regenerate them with the current encoder. tests/msgpack_compat.c asserts + * the values below. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(void) +{ + struct cmt *ctx = cmt_create(); + struct cmt_counter *counter; + struct cmt_counter *delta; + struct cmt_gauge *gauge; + struct cmt_gauge *signed_gauge; + struct cmt_gauge *unsigned_gauge; + struct cmt_untyped *untyped; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *metric; + double quantiles[] = {0.5, 0.9}; + double quantile_values[] = {0.2, 0.8}; + uint64_t buckets[] = {1, 3, 4, 5}; + uint64_t positive[] = {1, 2}; + uint64_t negative[] = {3}; + char *buffer; + size_t size; + + cmt_label_add(ctx, "cluster", "prod"); + + counter = cmt_counter_create(ctx, "fixture", "http", "requests_total", "requests", + 1, (char *[]) {"method"}); + cmt_counter_set(counter, 1000, 10, 1, (char *[]) {"GET"}); + cmt_counter_set(counter, 1000, 4, 1, (char *[]) {"POST"}); + metric = cmt_map_metric_get(&counter->opts, counter->map, 1, (char *[]) {"GET"}, 0); + cmt_metric_set_start_timestamp(metric, 500); + + delta = cmt_counter_create(ctx, "", "", "fixture_delta_total", "delta", 0, NULL); + delta->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + cmt_counter_allow_reset(delta); + cmt_counter_set(delta, 1000, 3, 0, NULL); + + gauge = cmt_gauge_create(ctx, "", "", "fixture_temperature", "temperature", + 1, (char *[]) {"room"}); + cmt_gauge_set(gauge, 1000, -12.5, 1, (char *[]) {"lab"}); + cmt_gauge_set(gauge, 1000, 21, 1, (char *[]) {""}); + + signed_gauge = cmt_gauge_create(ctx, "", "", "fixture_signed", "signed", 0, NULL); + cmt_gauge_set(signed_gauge, 1000, 0, 0, NULL); + cmt_metric_set_int64(&signed_gauge->map->metric, 1000, -9007199254740993LL); + + unsigned_gauge = cmt_gauge_create(ctx, "", "", "fixture_unsigned", "unsigned", 0, NULL); + cmt_gauge_set(unsigned_gauge, 1000, 0, 0, NULL); + cmt_metric_set_uint64(&unsigned_gauge->map->metric, 1000, UINT64_MAX); + + untyped = cmt_untyped_create(ctx, "", "", "fixture_untyped", "untyped", + 1, (char *[]) {"k"}); + cmt_untyped_set(untyped, 1000, 1.5, 1, (char *[]) {"v"}); + + histogram = cmt_histogram_create(ctx, "", "", "fixture_latency_seconds", "latency", + cmt_histogram_buckets_create(3, 0.1, 1.0, 10.0), + 1, (char *[]) {"route"}); + histogram->aggregation_type = CMT_AGGREGATION_TYPE_CUMULATIVE; + histogram->map->unit = cfl_sds_create("s"); + cmt_histogram_set_default(histogram, 1000, buckets, 6.5, 5, 1, (char *[]) {"/"}); + + summary = cmt_summary_create(ctx, "", "", "fixture_rpc_seconds", "rpc", 2, quantiles, + 1, (char *[]) {"svc"}); + cmt_summary_set_default(summary, 1000, quantile_values, 10, 20, 1, (char *[]) {"api"}); + + exp_histogram = cmt_exp_histogram_create(ctx, "", "", "fixture_exp", "exp", + 1, (char *[]) {"op"}); + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + cmt_exp_histogram_set_default(exp_histogram, 1000, 2, 1, 0.001, -1, 2, positive, + 0, 1, negative, 1, 4.5, 7, 1, (char *[]) {"read"}); + + if (cmt_encode_msgpack_create(ctx, &buffer, &size) != 0) { + return 1; + } + fwrite(buffer, 1, size, stdout); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + return 0; +} diff --git a/tests/data/all-types-v5.0.0.msgpack b/tests/data/all-types-v5.0.0.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..52f1f32233c0b49cea2860fb0c56a2263e6a1b02 GIT binary patch literal 1738 zcmb7EO=uHA6wW3s3JMnKK?0uCgY?jYe-~wgB6=wn)SIwQcPGix&F*$)Hncs}{v6bc zsS4JEG|A?tt!)De)l&<4k+cLV;zg)M1O*XA4<3X#nb}FRO)NT>ef##kH{Z|9j=Bj* zz*teG52UMjqzDHIL?$psW^H7t5M$kp-I&J&5M5Oa&<72ink=Y>gK3>hYqWybJ2lFN zO=Oc01zj>^!+>kq1dnG-T!`v}#6d8Fu^T7Ec1@6g3kVK45EHBrivYvbAXbYam-qI{ z)9G_fn1Iy zJUDlnC`lfr)R-UEZVrn<4{!wITOvz!ky{`jFkwRkYSJ@#=Nb}178hAoLa6CiWaX$| zkt{WZE4aD(==OnhCG4+{obcCA^Wr1X>BF*twH_6H4ud5qXw*EuNk^tkF4RDI$ZD&B|zBK^~KIVWNhN!~vRI7hM z)Yih<#Vtg&Yryp>I3{}$mZtlKM1FtdUw(hmNp8^7{D#^2w)P(R+tRuv3Y z%hWJ)DN7RqK^7;~nE?lwL^tRqEz9ru&Sm+_YHh7;16aULA_36Q_>V`aFBT(AYXB_Xc?2sG6pAHjl`6eM%V zatJK+Ich&7_-Ofy9{Tz9)s++G|Y8kuaBvvuhWd zrMo$uGavRmxW9g}zv(iyx1cj8=)ggndqe{mmp=E{9?C3?IAJwe$z#gae!a literal 0 HcmV?d00001 diff --git a/tests/data/all-types-v5.0.11.msgpack b/tests/data/all-types-v5.0.11.msgpack new file mode 100644 index 0000000000000000000000000000000000000000..20fe5c0e8f9f4c8dbd79eeae667e0b0a23f668f2 GIT binary patch literal 1793 zcmb7EOK1~8815!53JMnKK?0uCgY?jYuZyxl5xo=(>P=XuyOU(;W_LR?8`_>~9|!f) zR0Zon^4L7u+BTq2J++`0NlTz2UIZ;7D2OO}@F2w5ncZ!+iBjj1*+2h3|M&Q3#_S{{ zV7x5TAEe57vJ2S=( zo6M&m3fW}Xgl)EA5)I!@;ih1cHgDTZo${StB2mMgg^d+XofP(Fm{h!%8~*# z0vae5SRlY}s)?B35=)N%p}uSvvn{M3v4Ccya|D6MqFA%ci(xPm1sKt~Wh5n}$E! z0*62R$dn}vWkXjm^DR#UfzeOraY~*Yw17@jjb7xc_@4K#ieJ{8wZ08z30uh|K*JLY zei7|JgoZO};>#IlJ*bJF`KBNnRD`ZPt*tdZ2*6Z1#rVPQ-WNYsN17;xT2Nf}q}Bj| zx;z}9S#pDnY8HEMPXQSgfu(*+9)JYjsfy1yysGF9!`5rXCIP2iZ)OuJCPo~A}g?C_dpdpN5zANJh8w{dZx>8^FOU^VZtfP=XuyOU(;W_LR?8`_>~AA)*m zs)F?(d2AkSZ5vRio?6h0q$N-hFM<{k6hss~co5?3%EU(J~w&5a~c0oimXkLyT2Der*90KvY@MKtI%Qdb%WQ7N&JJtx-zu-kC9W z*knEhQOG94CTz0}lip@T0)~szwjCvE?0= zKX>+$od7sdJGuMnN@UOJClh?HSKk?|xBD(!bT@qIyL{lyO8CP+h-N6m3}g4mr7S5> zBcOp|fh7X`rkaQeF0u4@5Zq1`?9r2us(V$SfAm=hhnJ{uR1p)+hVZjDT&p%4iyHz=*Eq> zr##KvVQ>w77UdA;*gUYY5$1WU>g4#*m-V-Gw*EuZiU-Nk8$gx)r)|JG^R)?9=plz? zF_QJnD{g3ST@Bidt7|0cfIbp(KP(w z7C8LjN2V-cC>y$hnQwWT2#kI@k5lsOpapcIYV<{}itl*;s`z=$S?hnmEMY5|1Za3- z!QVtX5TW6Wn)qVIS@&z=Cms}JgNo3Vr+aHn7XmOhf@aX30%5s#)yaJq2V~1eW?Oc>of8yDC2A@T#Ia4E}A^bRl>*UD|bPHlbo- z#1Tl2d*|H|!tEUn#cDLdHRQBAy!P21&hym!UH9&8oF8bivZDphd5m^s2Ci%9}OoG>({D)#mFZhV{lwPYGtC29&v zMl431u2=koE9A5gTGH8P!7pm7_zAZt9p-B6a0ZA?44;O*!Ev_{1PYf)(|SjaT +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "cmt_tests.h" + +static const char *release_fixtures[] = { + "all-types-v5.0.0.msgpack", + "all-types-v5.0.11.msgpack", + "all-types-v5.1.5.msgpack", +}; + +/* Fluent Bit v5.0.0 (CMetrics 2.0.5) does not know "unit", and Fluent Bit 5.0 + * releases write the aggregation type of counters only: their histograms and + * exponential histograms arrive without temporality. */ +#define RELEASE_FIXTURE_HAS_UNIT(index) ((index) > 0) +#define RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index) ((index) > 1) + +static cfl_sds_t read_fixture(const char *name) +{ + char path[1024]; + + snprintf(path, sizeof(path), "%s/%s", CMT_TESTS_DATA_PATH, name); + return read_file(path); +} + +static struct cmt_map *find_map(struct cmt *ctx, int type, const char *name) +{ + struct cfl_list *list; + struct cfl_list *head; + struct cmt_map *map; + + switch (type) { + case CMT_COUNTER: + list = &ctx->counters; + break; + case CMT_GAUGE: + list = &ctx->gauges; + break; + case CMT_UNTYPED: + list = &ctx->untypeds; + break; + case CMT_SUMMARY: + list = &ctx->summaries; + break; + case CMT_HISTOGRAM: + list = &ctx->histograms; + break; + default: + list = &ctx->exp_histograms; + break; + } + + cfl_list_foreach(head, list) { + /* Every family struct starts with its opts and map pointers in the + * same order after a type-specific header; use the typed entries. */ + switch (type) { + case CMT_COUNTER: + map = cfl_list_entry(head, struct cmt_counter, _head)->map; + break; + case CMT_GAUGE: + map = cfl_list_entry(head, struct cmt_gauge, _head)->map; + break; + case CMT_UNTYPED: + map = cfl_list_entry(head, struct cmt_untyped, _head)->map; + break; + case CMT_SUMMARY: + map = cfl_list_entry(head, struct cmt_summary, _head)->map; + break; + case CMT_HISTOGRAM: + map = cfl_list_entry(head, struct cmt_histogram, _head)->map; + break; + default: + map = cfl_list_entry(head, struct cmt_exp_histogram, _head)->map; + break; + } + if (strcmp(map->opts->fqname, name) == 0) { + return map; + } + } + return NULL; +} + +static struct cmt_metric *find_sample(struct cmt_map *map, const char *label_value) +{ + struct cfl_list *head; + struct cmt_metric *sample; + struct cmt_map_label *label; + + if (label_value == NULL) { + return map->metric_static_set ? &map->metric : NULL; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + label = cfl_list_entry_first(&sample->labels, struct cmt_map_label, _head); + if (label != NULL && label->name != NULL && strcmp(label->name, label_value) == 0) { + return sample; + } + } + return NULL; +} + +static int has_static_label(struct cmt *ctx, const char *key, const char *value) +{ + struct cfl_list *head; + struct cmt_label *label; + + cfl_list_foreach(head, &ctx->static_labels->list) { + label = cfl_list_entry(head, struct cmt_label, _head); + if (strcmp(label->key, key) == 0 && strcmp(label->val, value) == 0) { + return CMT_TRUE; + } + } + return CMT_FALSE; +} + +/* The content written by tests/data/all-types-release.c. */ +static void check_release_content(struct cmt *ctx, int has_unit, int has_temporality, + const char *source) +{ + struct cmt_map *map; + struct cmt_metric *sample; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_exp_histogram_snapshot snapshot; + + TEST_CASE(source); + TEST_CHECK(has_static_label(ctx, "cluster", "prod")); + + map = find_map(ctx, CMT_COUNTER, "fixture_http_requests_total"); + TEST_ASSERT(map != NULL); + TEST_CHECK(map->label_count == 1); + sample = find_sample(map, "GET"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 10); + TEST_CHECK(cmt_metric_get_timestamp(sample) == 1000); + TEST_CHECK(cmt_metric_has_start_timestamp(sample)); + TEST_CHECK(cmt_metric_get_start_timestamp(sample) == 500); + sample = find_sample(map, "POST"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 4); + TEST_CHECK(!cmt_metric_has_start_timestamp(sample)); + + map = find_map(ctx, CMT_COUNTER, "fixture_delta_total"); + TEST_ASSERT(map != NULL); + TEST_CHECK(((struct cmt_counter *) map->parent)->aggregation_type == + CMT_AGGREGATION_TYPE_DELTA); + TEST_CHECK(cmt_metric_get_value(&map->metric) == 3); + + map = find_map(ctx, CMT_GAUGE, "fixture_temperature"); + TEST_ASSERT(map != NULL); + sample = find_sample(map, "lab"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == -12.5); + sample = find_sample(map, ""); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 21); + + map = find_map(ctx, CMT_GAUGE, "fixture_signed"); + TEST_ASSERT(map != NULL); + TEST_CHECK(cmt_metric_get_value_type(&map->metric) == CMT_METRIC_VALUE_INT64); + TEST_CHECK(cmt_metric_get_int64_value(&map->metric) == -9007199254740993LL); + + map = find_map(ctx, CMT_GAUGE, "fixture_unsigned"); + TEST_ASSERT(map != NULL); + TEST_CHECK(cmt_metric_get_value_type(&map->metric) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK(cmt_metric_get_uint64_value(&map->metric) == UINT64_MAX); + + map = find_map(ctx, CMT_UNTYPED, "fixture_untyped"); + TEST_ASSERT(map != NULL); + sample = find_sample(map, "v"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 1.5); + + map = find_map(ctx, CMT_HISTOGRAM, "fixture_latency_seconds"); + TEST_ASSERT(map != NULL); + histogram = map->parent; + TEST_CHECK(histogram->aggregation_type == (has_temporality ? + CMT_AGGREGATION_TYPE_CUMULATIVE : + CMT_AGGREGATION_TYPE_UNSPECIFIED)); + TEST_ASSERT(histogram->buckets != NULL && histogram->buckets->count == 3); + TEST_CHECK(histogram->buckets->upper_bounds[0] == 0.1); + TEST_CHECK(histogram->buckets->upper_bounds[2] == 10.0); + if (has_unit) { + TEST_CHECK(map->unit != NULL && strcmp(map->unit, "s") == 0); + } + sample = find_sample(map, "/"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_hist_get_value(sample, 0) == 1); + TEST_CHECK(cmt_metric_hist_get_value(sample, 1) == 3); + TEST_CHECK(cmt_metric_hist_get_value(sample, 2) == 4); + TEST_CHECK(cmt_metric_hist_get_value(sample, 3) == 5); + TEST_CHECK(cmt_metric_hist_get_count_value(sample) == 5); + TEST_CHECK(cmt_metric_hist_get_sum_value(sample) == 6.5); + + map = find_map(ctx, CMT_SUMMARY, "fixture_rpc_seconds"); + TEST_ASSERT(map != NULL); + summary = map->parent; + TEST_ASSERT(summary->quantiles_count == 2); + TEST_CHECK(summary->quantiles[0] == 0.5 && summary->quantiles[1] == 0.9); + sample = find_sample(map, "api"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_summary_quantile_get_value(sample, 0) == 0.2); + TEST_CHECK(cmt_summary_quantile_get_value(sample, 1) == 0.8); + TEST_CHECK(cmt_summary_get_sum_value(sample) == 10); + TEST_CHECK(cmt_summary_get_count_value(sample) == 20); + + map = find_map(ctx, CMT_EXP_HISTOGRAM, "fixture_exp"); + TEST_ASSERT(map != NULL); + exp_histogram = map->parent; + TEST_CHECK(exp_histogram->aggregation_type == (has_temporality ? + CMT_AGGREGATION_TYPE_DELTA : + CMT_AGGREGATION_TYPE_UNSPECIFIED)); + sample = find_sample(map, "read"); + TEST_ASSERT(sample != NULL); + TEST_ASSERT(cmt_metric_exp_hist_get_snapshot(sample, &snapshot) == 0); + TEST_CHECK(snapshot.scale == 2); + TEST_CHECK(snapshot.zero_count == 1); + TEST_CHECK(snapshot.zero_threshold == 0.001); + TEST_CHECK(snapshot.positive_offset == -1 && snapshot.positive_count == 2); + TEST_CHECK(snapshot.positive_count == 2 && + snapshot.positive_buckets[0] == 1 && snapshot.positive_buckets[1] == 2); + TEST_CHECK(snapshot.negative_offset == 0 && snapshot.negative_count == 1); + TEST_CHECK(snapshot.negative_count == 1 && snapshot.negative_buckets[0] == 3); + TEST_CHECK(snapshot.count == 7); + TEST_CHECK(snapshot.sum_set); + cmt_metric_exp_hist_snapshot_destroy(&snapshot); +} + +static struct cmt *decode_all(char *buffer, size_t size, size_t *offset) +{ + struct cmt *decoded = NULL; + int result; + + result = cmt_decode_msgpack_create(&decoded, buffer, size, offset); + TEST_CHECK_(result == CMT_DECODE_MSGPACK_SUCCESS, "decode result %d", result); + return result == CMT_DECODE_MSGPACK_SUCCESS ? decoded : NULL; +} + +/* Chunks written by released agents decode completely in this release, keep + * their content through a re-encode (forward relay) and through cmt_cat. */ +static void test_release_fixtures(void) +{ + size_t index; + size_t offset; + size_t size; + cfl_sds_t fixture; + struct cmt *decoded; + struct cmt *again; + struct cmt *merged; + char *buffer; + char label[128]; + + for (index = 0; index < sizeof(release_fixtures) / sizeof(*release_fixtures); index++) { + fixture = read_fixture(release_fixtures[index]); + TEST_ASSERT(fixture != NULL); + + offset = 0; + decoded = decode_all(fixture, cfl_sds_len(fixture), &offset); + TEST_ASSERT(decoded != NULL); + TEST_CHECK(offset == cfl_sds_len(fixture)); + snprintf(label, sizeof(label), "%s decoded", release_fixtures[index]); + check_release_content(decoded, RELEASE_FIXTURE_HAS_UNIT(index), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index), label); + + TEST_ASSERT(cmt_encode_msgpack_create(decoded, &buffer, &size) == 0); + offset = 0; + again = decode_all(buffer, size, &offset); + TEST_ASSERT(again != NULL); + snprintf(label, sizeof(label), "%s re-encoded", release_fixtures[index]); + check_release_content(again, RELEASE_FIXTURE_HAS_UNIT(index), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index), label); + + merged = cmt_create(); + TEST_ASSERT(merged != NULL); + TEST_CHECK(cmt_cat(merged, again) == 0); + snprintf(label, sizeof(label), "%s concatenated", release_fixtures[index]); + check_release_content(merged, RELEASE_FIXTURE_HAS_UNIT(index), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index), label); + + cmt_destroy(merged); + cmt_decode_msgpack_destroy(again); + cmt_encode_msgpack_destroy(buffer); + cmt_decode_msgpack_destroy(decoded); + cfl_sds_destroy(fixture); + } +} + +/* A forward chunk can hold contexts from agents of different releases. */ +static void test_mixed_release_batch(void) +{ + size_t index; + size_t offset; + size_t count; + size_t size; + cfl_sds_t batch; + cfl_sds_t fixture; + struct cmt *decoded; + char *buffer; + + batch = cfl_sds_create_size(0); + TEST_ASSERT(batch != NULL); + for (index = 0; index < sizeof(release_fixtures) / sizeof(*release_fixtures); index++) { + fixture = read_fixture(release_fixtures[index]); + TEST_ASSERT(fixture != NULL); + batch = cfl_sds_cat(batch, fixture, cfl_sds_len(fixture)); + cfl_sds_destroy(fixture); + } + + /* The current encoder's output for the newest fixture closes the batch. */ + fixture = read_fixture(release_fixtures[2]); + offset = 0; + decoded = decode_all(fixture, cfl_sds_len(fixture), &offset); + TEST_ASSERT(decoded != NULL); + TEST_ASSERT(cmt_encode_msgpack_create(decoded, &buffer, &size) == 0); + batch = cfl_sds_cat(batch, buffer, size); + cmt_encode_msgpack_destroy(buffer); + cmt_decode_msgpack_destroy(decoded); + cfl_sds_destroy(fixture); + + offset = 0; + count = 0; + while (offset < cfl_sds_len(batch)) { + decoded = decode_all(batch, cfl_sds_len(batch), &offset); + TEST_ASSERT(decoded != NULL); + check_release_content(decoded, RELEASE_FIXTURE_HAS_UNIT(count), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(count), "batch entry"); + cmt_decode_msgpack_destroy(decoded); + count++; + } + TEST_CHECK(count == 4); + TEST_CHECK(offset == cfl_sds_len(batch)); + cfl_sds_destroy(batch); +} + +/* Every metric type with labels, static samples, typed values, start + * timestamps, units, OTLP metadata and the native metadata sidecar. */ +static struct cmt *create_full_context(void) +{ + struct cmt *ctx; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *sample; + struct cfl_kvlist *metadata; + double quantiles[] = {0.5, 0.99}; + double quantile_values[] = {1, 2}; + uint64_t buckets[] = {1, 2, 3}; + uint64_t positive[] = {4, 5}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + TEST_ASSERT(cmt_label_add(ctx, "dc", "eu-1") == 0); + + counter = cmt_counter_create(ctx, "ns", "sub", "events_total", "events", 2, + (char *[]) {"a", "b"}); + TEST_ASSERT(counter != NULL); + cmt_counter_allow_reset(counter); + counter->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + TEST_ASSERT(cmt_counter_set(counter, 10, 5, 2, (char *[]) {"x", "y"}) == 0); + sample = cmt_map_metric_get(&counter->opts, counter->map, 2, (char *[]) {"x", "y"}, CMT_FALSE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_start_timestamp(sample, 5); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", 1) == 0); + counter->map->unit = cfl_sds_create("{event}"); + + gauge = cmt_gauge_create(ctx, "", "", "level", "level", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 10, 0, 0, NULL) == 0); + cmt_metric_set_int64(&gauge->map->metric, 10, INT64_MIN); + gauge->map->otlp_metadata = cfl_kvlist_create(); + metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(metadata, "k", "v") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", metadata) == 0); + + untyped = cmt_untyped_create(ctx, "", "", "raw", "raw", 1, (char *[]) {"k"}); + TEST_ASSERT(untyped != NULL); + TEST_ASSERT(cmt_untyped_set(untyped, 10, NAN, 1, (char *[]) {"nan"}) == 0); + + histogram = cmt_histogram_create(ctx, "", "", "latency", "latency", + cmt_histogram_buckets_create(2, 1.0, 2.0), + 1, (char *[]) {"r"}); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_set_default(histogram, 10, buckets, 4.5, 3, 1, + (char *[]) {"/"}) == 0); + + summary = cmt_summary_create(ctx, "", "", "rpc", "rpc", 2, quantiles, 0, NULL); + TEST_ASSERT(summary != NULL); + TEST_ASSERT(cmt_summary_set_default(summary, 10, quantile_values, 3, 2, 0, NULL) == 0); + + exp_histogram = cmt_exp_histogram_create(ctx, "", "", "exp", "exp", 0, NULL); + TEST_ASSERT(exp_histogram != NULL); + TEST_ASSERT(cmt_exp_histogram_set_default(exp_histogram, 10, -2, 0, 0, 3, 2, positive, + 0, 0, NULL, 0, 0, 9, 0, NULL) == 0); + return ctx; +} + +/* Released decoders (Fluent Bit 5.0.11, 5.1.x) accept exactly these keys; + * Fluent Bit 5.0.0 accepts all of them except "unit". */ +static const char *released_keys[] = { + "aggregation_type", "buckets", "cmetrics", "count", "desc", "exp_histogram", + "external", "hash", "histogram", "labels", "meta", "metrics", "name", + "negative_buckets", "negative_offset", "ns", "opts", "positive_buckets", + "positive_offset", "processing", "quantiles", "quantiles_set", "scale", "ss", + "start_ts", "static_labels", "sum", "summary", "sum_set", "ts", "type", "unit", + "value", "values", "value_type", "value_int64", "value_uint64", "ver", + "zero_count", "zero_threshold", NULL +}; + +static int is_released_key(const char *key, size_t length) +{ + size_t index; + + for (index = 0; released_keys[index] != NULL; index++) { + if (strlen(released_keys[index]) == length && + memcmp(released_keys[index], key, length) == 0) { + return CMT_TRUE; + } + } + return CMT_FALSE; +} + +/* Walk a value; map keys are schema keys except inside the schemaless + * internal ("cmetrics") and external metadata maps, which are skipped. */ +static int walk_keys(mpack_reader_t *reader, int depth) +{ + mpack_tag_t tag; + uint32_t index; + uint32_t count; + char key[64]; + uint32_t length; + int unknown = 0; + + tag = mpack_read_tag(reader); + if (mpack_reader_error(reader) != mpack_ok) { + return -1; + } + switch (mpack_tag_type(&tag)) { + case mpack_type_map: + count = mpack_tag_map_count(&tag); + for (index = 0; index < count; index++) { + tag = mpack_read_tag(reader); + if (mpack_tag_type(&tag) != mpack_type_str) { + return -1; + } + length = mpack_tag_str_length(&tag); + if (length >= sizeof(key)) { + return -1; + } + mpack_read_bytes(reader, key, length); + mpack_done_str(reader); + if (!is_released_key(key, length)) { + TEST_MSG("key not known to released readers: %.*s", (int) length, key); + unknown++; + } + if ((length == 8 && memcmp(key, "cmetrics", 8) == 0) || + (length == 8 && memcmp(key, "external", 8) == 0)) { + mpack_discard(reader); + } + else { + unknown += walk_keys(reader, depth + 1); + } + } + mpack_done_map(reader); + break; + case mpack_type_array: + count = mpack_tag_array_count(&tag); + for (index = 0; index < count; index++) { + unknown += walk_keys(reader, depth + 1); + } + mpack_done_array(reader); + break; + case mpack_type_str: + mpack_skip_bytes(reader, mpack_tag_str_length(&tag)); + mpack_done_str(reader); + break; + case mpack_type_bin: + mpack_skip_bytes(reader, mpack_tag_bin_length(&tag)); + mpack_done_bin(reader); + break; + default: + break; + } + return unknown; +} + +/* New information must travel in the schemaless metadata maps: a new schema + * key makes every released reader reject the whole chunk. */ +static void test_encoder_uses_released_keys_only(void) +{ + struct cmt *ctx; + mpack_reader_t reader; + char *buffer; + size_t size; + int unknown; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + mpack_reader_init_data(&reader, buffer, size); + unknown = walk_keys(&reader, 0); + TEST_CHECK(mpack_reader_destroy(&reader) == mpack_ok); + TEST_CHECK_(unknown == 0, "%d keys unknown to released readers", unknown); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static uint64_t double_bits(double value) +{ + uint64_t bits; + + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +/* Exact values of every scalar type for counters, gauges and untyped. */ +static void test_scalar_extremes(void) +{ + static const uint64_t double_patterns[] = { + 0x7ff8000000000000ULL, /* quiet NaN */ + 0x7ff0000000000002ULL, /* Prometheus staleness marker */ + 0x7ff0000000000000ULL, /* +Inf */ + 0xfff0000000000000ULL, /* -Inf */ + 0x8000000000000000ULL, /* -0.0 */ + 0x0000000000000001ULL, /* smallest denormal */ + 0x7fefffffffffffffULL, /* DBL_MAX */ + }; + static const int64_t signed_values[] = {INT64_MIN, INT64_MAX, -1, 0}; + static const uint64_t unsigned_values[] = {UINT64_MAX, (uint64_t) INT64_MAX + 1, 0}; + struct cmt *ctx; + struct cmt *decoded; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_metric *sample; + struct cmt_map *maps[3]; + char value_label[32]; + char *labels[] = {value_label}; + char *buffer; + size_t size; + size_t offset; + size_t index; + size_t type; + double value; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + counter = cmt_counter_create(ctx, "", "", "c", "c", 1, (char *[]) {"v"}); + gauge = cmt_gauge_create(ctx, "", "", "g", "g", 1, (char *[]) {"v"}); + untyped = cmt_untyped_create(ctx, "", "", "u", "u", 1, (char *[]) {"v"}); + TEST_ASSERT(counter != NULL && gauge != NULL && untyped != NULL); + maps[0] = counter->map; + maps[1] = gauge->map; + maps[2] = untyped->map; + + for (type = 0; type < 3; type++) { + for (index = 0; index < sizeof(double_patterns) / sizeof(*double_patterns); index++) { + snprintf(value_label, sizeof(value_label), "d%zu", index); + sample = cmt_map_metric_get(&counter->opts, maps[type], 1, labels, CMT_TRUE); + TEST_ASSERT(sample != NULL); + memcpy(&value, &double_patterns[index], sizeof(value)); + cmt_metric_set_double(sample, 1, value); + } + for (index = 0; index < sizeof(signed_values) / sizeof(*signed_values); index++) { + snprintf(value_label, sizeof(value_label), "i%zu", index); + sample = cmt_map_metric_get(&counter->opts, maps[type], 1, labels, CMT_TRUE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_int64(sample, 1, signed_values[index]); + } + for (index = 0; index < sizeof(unsigned_values) / sizeof(*unsigned_values); index++) { + snprintf(value_label, sizeof(value_label), "u%zu", index); + sample = cmt_map_metric_get(&counter->opts, maps[type], 1, labels, CMT_TRUE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_uint64(sample, 1, unsigned_values[index]); + } + } + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + offset = 0; + decoded = decode_all(buffer, size, &offset); + TEST_ASSERT(decoded != NULL); + maps[0] = find_map(decoded, CMT_COUNTER, "c"); + maps[1] = find_map(decoded, CMT_GAUGE, "g"); + maps[2] = find_map(decoded, CMT_UNTYPED, "u"); + for (type = 0; type < 3; type++) { + TEST_ASSERT(maps[type] != NULL); + for (index = 0; index < sizeof(double_patterns) / sizeof(*double_patterns); index++) { + snprintf(value_label, sizeof(value_label), "d%zu", index); + sample = find_sample(maps[type], value_label); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_DOUBLE); + TEST_CHECK_(double_bits(cmt_metric_get_value(sample)) == double_patterns[index], + "type %zu double %016" PRIx64, type, double_patterns[index]); + } + for (index = 0; index < sizeof(signed_values) / sizeof(*signed_values); index++) { + snprintf(value_label, sizeof(value_label), "i%zu", index); + sample = find_sample(maps[type], value_label); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_INT64); + TEST_CHECK_(cmt_metric_get_int64_value(sample) == signed_values[index], + "type %zu int64 %" PRId64, type, signed_values[index]); + } + for (index = 0; index < sizeof(unsigned_values) / sizeof(*unsigned_values); index++) { + snprintf(value_label, sizeof(value_label), "u%zu", index); + sample = find_sample(maps[type], value_label); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK_(cmt_metric_get_uint64_value(sample) == unsigned_values[index], + "type %zu uint64 %" PRIu64, type, unsigned_values[index]); + } + } + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Storage corruption or a broken peer: every prefix and every single byte + * corruption of a full chunk is rejected or decoded, never crashes, and a + * decoded context can be encoded again. Run under Valgrind or ASan. */ +static void test_corrupted_chunks(void) +{ + struct cmt *ctx; + struct cmt *decoded; + char *buffer; + char *copy; + char *again; + size_t size; + size_t again_size; + size_t length; + size_t index; + size_t offset; + int result; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + copy = malloc(size); + TEST_ASSERT(copy != NULL); + + for (length = 0; length < size; length++) { + offset = 0; + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, buffer, length, &offset); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + cmt_decode_msgpack_destroy(decoded); + } + } + + for (index = 0; index < size; index++) { + memcpy(copy, buffer, size); + copy[index] ^= 0xa5; + offset = 0; + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, copy, size, &offset); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + TEST_CHECK(offset <= size); + if (cmt_encode_msgpack_create(decoded, &again, &again_size) == 0) { + cmt_encode_msgpack_destroy(again); + } + cmt_decode_msgpack_destroy(decoded); + } + } + + free(copy); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Replace the array header that follows the nth occurrence of a key. */ +static char *replace_array_header(char *buffer, size_t size, const char *key, + const char *header, size_t header_size, size_t *out_size) +{ + size_t index; + size_t key_length; + char *patched; + char encoded_key[32]; + + key_length = strlen(key); + encoded_key[0] = (char) (0xa0 | key_length); + memcpy(&encoded_key[1], key, key_length); + for (index = 0; index + key_length + 2 <= size; index++) { + if (memcmp(&buffer[index], encoded_key, key_length + 1) != 0) { + continue; + } + index += key_length + 1; + /* fixarray headers only: the fixture arrays are small */ + if (((unsigned char) buffer[index] & 0xf0) != 0x90) { + return NULL; + } + patched = malloc(size + header_size); + TEST_ASSERT(patched != NULL); + memcpy(patched, buffer, index); + memcpy(&patched[index], header, header_size); + memcpy(&patched[index + header_size], &buffer[index + 1], size - index - 1); + *out_size = size + header_size - 1; + return patched; + } + return NULL; +} + +/* A declared element count far beyond the remaining input must be rejected + * before any allocation sized from it. */ +static void test_oversized_declared_counts(void) +{ + static const char *keys[] = {"values", "labels", "buckets", "quantiles"}; + static const char huge_array[] = {(char) 0xdd, (char) 0xff, (char) 0xff, + (char) 0xff, (char) 0xff}; + struct cmt *ctx; + struct cmt *decoded; + char *buffer; + char *patched; + size_t size; + size_t patched_size; + size_t offset; + size_t index; + int result; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + for (index = 0; index < sizeof(keys) / sizeof(*keys); index++) { + patched = replace_array_header(buffer, size, keys[index], huge_array, + sizeof(huge_array), &patched_size); + TEST_ASSERT_(patched != NULL, "array key %s found", keys[index]); + offset = 0; + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, patched, patched_size, &offset); + TEST_CHECK_(result != CMT_DECODE_MSGPACK_SUCCESS, "%s count rejected", keys[index]); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + cmt_decode_msgpack_destroy(decoded); + } + free(patched); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Released readers reject any other format version, so the version cannot + * be raised to signal new content; this release must behave the same. */ +static void test_format_version_is_strict(void) +{ + static const char version_key[] = {(char) 0xa3, 'v', 'e', 'r', 0x02}; + struct cmt *ctx; + struct cmt *decoded; + char *buffer; + size_t size; + size_t index; + size_t offset; + int found = CMT_FALSE; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + for (index = 0; index + sizeof(version_key) <= size; index++) { + if (memcmp(&buffer[index], version_key, sizeof(version_key)) == 0) { + buffer[index + sizeof(version_key) - 1] = 0x03; + found = CMT_TRUE; + break; + } + } + TEST_ASSERT(found); + offset = 0; + decoded = NULL; + TEST_CHECK(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) != + CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(decoded == NULL); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +TEST_LIST = { + {"release_fixtures", test_release_fixtures}, + {"mixed_release_batch", test_mixed_release_batch}, + {"encoder_uses_released_keys_only", test_encoder_uses_released_keys_only}, + {"scalar_extremes", test_scalar_extremes}, + {"corrupted_chunks", test_corrupted_chunks}, + {"oversized_declared_counts", test_oversized_declared_counts}, + {"format_version_is_strict", test_format_version_is_strict}, + {0} +};