diff --git a/src/cmt_cat.c b/src/cmt_cat.c index 40e09012..aba564ad 100644 --- a/src/cmt_cat.c +++ b/src/cmt_cat.c @@ -506,6 +506,8 @@ static int copy_point_metadata(struct cmt_metric *dst, struct cmt_metric *src) struct cfl_kvlist *copy; if (src->otlp_metadata == NULL) { + cfl_kvlist_destroy(dst->otlp_metadata); + dst->otlp_metadata = NULL; return 0; } @@ -620,6 +622,20 @@ static struct cmt_metric *get_point_sample(struct cmt_opts *opts, struct cmt_map struct cfl_list *head; struct cmt_metric *metric; + if (count > 0 && count != dst->label_count) { + /* OTLP points of one metric may carry fewer attributes than the + * family has label keys. The index only holds complete label sets, + * so such samples are matched by value and appended. */ + cfl_list_foreach(head, &dst->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (sample_labels_match(metric, count, labels) && + (src->otlp_metadata == NULL || same_data_point(metric, src))) { + return metric; + } + } + return append_point_sample(dst, src, count, labels); + } + if (count == 0) { /* Label-less points always merge into the static metric. */ return cmt_map_metric_get(opts, dst, 0, NULL, CMT_TRUE); @@ -646,6 +662,87 @@ static struct cmt_metric *get_point_sample(struct cmt_opts *opts, struct cmt_map return append_point_sample(dst, src, count, labels); } +/* + * Unit and static sample state describe a family. They are set when cmt_cat() + * creates it, and again for each contributor while it holds no samples, then + * never changed: later contributors cannot reinterpret samples already + * merged. Label-less families are created with their static sample set: a + * new destination must not report a sample its source never had. + */ +static int init_family(struct cmt_map *dst, struct cmt_map *family, + struct cmt_map *points) +{ + if (dst->unit != NULL) { + cfl_sds_destroy(dst->unit); + dst->unit = NULL; + } + if (family->unit != NULL) { + dst->unit = cfl_sds_create_len(family->unit, cfl_sds_len(family->unit)); + if (dst->unit == NULL) { + return -1; + } + } + if (!points->metric_static_set) { + dst->metric_static_set = CMT_FALSE; + } + return 0; +} + +static int map_has_samples(struct cmt_map *map) +{ + return map->metric_static_set || cfl_list_size(&map->metrics) > 0; +} + +/* Unspecified temporality (written by releases that did not record it) is + * compatible with either. */ +static int aggregation_compatible(int dst, int src) +{ + return dst == src || + dst == CMT_AGGREGATION_TYPE_UNSPECIFIED || + src == CMT_AGGREGATION_TYPE_UNSPECIFIED; +} + +/* An empty unit is unspecified and compatible with any. Values are never + * converted, so two explicit units must be equal. */ +static int units_compatible(struct cmt_map *dst, struct cmt_map *src) +{ + if (dst->unit == NULL || src->unit == NULL || + cfl_sds_len(dst->unit) == 0 || cfl_sds_len(src->unit) == 0) { + return CMT_TRUE; + } + return cfl_sds_len(dst->unit) == cfl_sds_len(src->unit) && + memcmp(dst->unit, src->unit, cfl_sds_len(src->unit)) == 0; +} + +static int label_keys_match(struct cmt_map *left, + struct cmt_map *right) +{ + struct cfl_list *left_head; + struct cfl_list *right_head; + struct cmt_map_label *left_label; + struct cmt_map_label *right_label; + + left_head = left->label_keys.next; + right_head = right->label_keys.next; + + while (left_head != &left->label_keys && + right_head != &right->label_keys) { + left_label = cfl_list_entry(left_head, struct cmt_map_label, _head); + right_label = cfl_list_entry(right_head, struct cmt_map_label, _head); + + if (left_label->name == NULL || right_label->name == NULL || + strcmp(left_label->name, right_label->name) != 0) { + return CMT_FALSE; + } + + left_head = left_head->next; + right_head = right_head->next; + } + + return left_head == &left->label_keys && + right_head == &right->label_keys; +} + int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map *src) { int c; @@ -658,6 +755,20 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map struct cmt_summary *summary; struct cmt_histogram *histogram_src; struct cmt_histogram *histogram_dst; + int same_keys; + + /* Only OTLP points of the same family carry fewer values than its label + * keys; any other mismatch has no meaning here. Check every sample before + * changing the destination. */ + same_keys = label_keys_match(dst, src); + cfl_list_foreach(head, &src->metrics) { + metric_src = cfl_list_entry(head, struct cmt_metric, _head); + c = cfl_list_size(&metric_src->labels); + if (c > 0 && c != dst->label_count && + (c > dst->label_count || !same_keys)) { + return -1; + } + } if (src->otlp_metadata) { metadata = cmt_otlp_metadata_clone(src->otlp_metadata); @@ -714,7 +825,7 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map } c = cfl_list_size(&metric_src->labels); - if (metric_src->otlp_metadata != NULL) { + if (metric_src->otlp_metadata != NULL || (c > 0 && c != dst->label_count)) { metric_dst = get_point_sample(opts, dst, metric_src, c, labels); } else { @@ -870,40 +981,12 @@ static struct cmt_summary *summary_lookup(struct cmt *cmt, struct cmt_opts *opts return NULL; } -static int summary_label_keys_match(struct cmt_map *left, - struct cmt_map *right) -{ - struct cfl_list *left_head; - struct cfl_list *right_head; - struct cmt_map_label *left_label; - struct cmt_map_label *right_label; - - left_head = left->label_keys.next; - right_head = right->label_keys.next; - - while (left_head != &left->label_keys && - right_head != &right->label_keys) { - left_label = cfl_list_entry(left_head, struct cmt_map_label, _head); - right_label = cfl_list_entry(right_head, struct cmt_map_label, _head); - - if (left_label->name == NULL || right_label->name == NULL || - strcmp(left_label->name, right_label->name) != 0) { - return CMT_FALSE; - } - - left_head = left_head->next; - right_head = right_head->next; - } - - return left_head == &left->label_keys && - right_head == &right->label_keys; -} - int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, struct cmt_map *filtered_map) { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_counter *c; @@ -923,6 +1006,7 @@ int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -930,6 +1014,22 @@ int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, return -1; } + /* Temporality and monotonicity are family semantics: a contributor + * with samples must not change how existing samples are read. */ + if (created || !map_has_samples(c->map)) { + if (init_family(c->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + c->aggregation_type = counter->aggregation_type; + c->allow_reset = counter->allow_reset; + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + (!aggregation_compatible(c->aggregation_type, counter->aggregation_type) || + c->allow_reset != counter->allow_reset || + !units_compatible(c->map, map))) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&c->opts, c->map, filtered_map); if (ret == -1) { @@ -951,6 +1051,7 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_gauge *g; @@ -970,6 +1071,7 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -977,6 +1079,16 @@ int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, return -1; } + if (created || !map_has_samples(g->map)) { + if (init_family(g->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + !units_compatible(g->map, map)) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&g->opts, g->map, filtered_map); if (ret == -1) { @@ -998,6 +1110,7 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_untyped *u; @@ -1017,6 +1130,7 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -1024,6 +1138,16 @@ int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, return -1; } + if (created || !map_has_samples(u->map)) { + if (init_family(u->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + !units_compatible(u->map, map)) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&u->opts, u->map, filtered_map); if (ret == -1) { @@ -1045,6 +1169,7 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_histogram *hist; @@ -1076,6 +1201,7 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, opts->name, opts->description, buckets, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -1083,6 +1209,19 @@ int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, return -1; } + if (created || !map_has_samples(hist->map)) { + if (init_family(hist->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + hist->aggregation_type = histogram->aggregation_type; + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + (!aggregation_compatible(hist->aggregation_type, + histogram->aggregation_type) || + !units_compatible(hist->map, map))) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&hist->opts, hist->map, filtered_map); if (ret == -1) { @@ -1105,6 +1244,7 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, size_t i; int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_summary *sum; @@ -1119,7 +1259,7 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, sum = summary_lookup(cmt, opts); if (sum != NULL) { - if (!summary_label_keys_match(sum->map, map)) { + if (!label_keys_match(sum->map, map)) { free(labels); return -1; } @@ -1154,6 +1294,7 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, summary->quantiles_count, quantiles, map->label_count, labels); + created = CMT_TRUE; free(quantiles); } @@ -1162,6 +1303,16 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, return -1; } + if (created || !map_has_samples(sum->map)) { + if (init_family(sum->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + !units_compatible(sum->map, map)) { + return -1; + } + if (filtered_map != NULL) { ret = cmt_cat_copy_map(&sum->opts, sum->map, filtered_map); if (ret == -1) { @@ -1183,6 +1334,7 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr { int ret; char **labels = NULL; + int created = CMT_FALSE; struct cmt_map *map; struct cmt_opts *opts; struct cmt_exp_histogram *eh; @@ -1201,6 +1353,7 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr opts->ns, opts->subsystem, opts->name, opts->description, map->label_count, labels); + created = CMT_TRUE; } free(labels); @@ -1208,7 +1361,18 @@ int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogr return -1; } - eh->aggregation_type = exp_histogram->aggregation_type; + if (created || !map_has_samples(eh->map)) { + if (init_family(eh->map, map, filtered_map != NULL ? filtered_map : map) != 0) { + return -1; + } + eh->aggregation_type = exp_histogram->aggregation_type; + } + else if (map_has_samples(filtered_map != NULL ? filtered_map : map) && + (!aggregation_compatible(eh->aggregation_type, + exp_histogram->aggregation_type) || + !units_compatible(eh->map, map))) { + return -1; + } if (filtered_map != NULL) { ret = cmt_cat_copy_map(&eh->opts, eh->map, filtered_map); diff --git a/src/cmt_decode_opentelemetry.c b/src/cmt_decode_opentelemetry.c index 2bb3e460..04210a15 100644 --- a/src/cmt_decode_opentelemetry.c +++ b/src/cmt_decode_opentelemetry.c @@ -38,7 +38,11 @@ #include #include -static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source); +/* Distinguish absent optional values from allocation failures. */ +#define CLONE_UNSUPPORTED_VALUE -1 + +static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source, + int *status); static int clone_array(struct cfl_array *target, Opentelemetry__Proto__Common__V1__ArrayValue *source); @@ -63,14 +67,18 @@ static int clone_exemplars_to_kvlist(struct cfl_kvlist *target, Opentelemetry__Proto__Metrics__V1__Exemplar **exemplars, size_t exemplar_count); -static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source) +static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source, + int *status) { struct cfl_kvlist *new_child_kvlist; struct cfl_array *new_child_array; struct cfl_variant *result_instance = NULL; int result; - if (source == NULL) { + *status = CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + if (source == NULL || source->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET) { + *status = CLONE_UNSUPPORTED_VALUE; return NULL; } if (source->value_case == OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE) { @@ -105,6 +113,7 @@ static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyVa result = clone_kvlist(new_child_kvlist, source->kvlist_value); if (result) { cfl_variant_destroy(result_instance); + *status = result; return NULL; } @@ -126,6 +135,7 @@ static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyVa result = clone_array(new_child_array, source->array_value); if (result) { cfl_variant_destroy(result_instance); + *status = result; return NULL; } @@ -134,6 +144,13 @@ static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyVa result_instance = cfl_variant_create_from_bytes((char *) source->bytes_value.data, source->bytes_value.len, CFL_FALSE); } + else { + *status = CLONE_UNSUPPORTED_VALUE; + } + + if (result_instance != NULL) { + *status = CMT_DECODE_OPENTELEMETRY_SUCCESS; + } return result_instance; } @@ -167,9 +184,9 @@ static int clone_array_entry(struct cfl_array *target, return CMT_DECODE_OPENTELEMETRY_SUCCESS; } - new_child_instance = clone_variant(source); + new_child_instance = clone_variant(source, &result); if (new_child_instance == NULL) { - return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + return result; } result = cfl_array_append(target, new_child_instance); @@ -214,10 +231,10 @@ static int clone_kvlist_entry(struct cfl_kvlist *target, return CMT_DECODE_OPENTELEMETRY_SUCCESS; } - new_child_instance = clone_variant(source->value); + new_child_instance = clone_variant(source->value, &result); if (new_child_instance == NULL) { - return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + return result; } result = cfl_kvlist_insert(target, source->key, new_child_instance); @@ -378,6 +395,7 @@ static int clone_point_attributes(struct cfl_kvlist *metadata, { struct cfl_kvlist *copy; size_t index; + int result; copy = cfl_kvlist_create(); if (copy == NULL) { @@ -390,7 +408,14 @@ static int clone_point_attributes(struct cfl_kvlist *metadata, OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET)) { continue; } - if (attributes[index] == NULL || clone_kvlist_entry(copy, attributes[index]) != 0) { + result = clone_kvlist_entry(copy, attributes[index]); + if (result == CLONE_UNSUPPORTED_VALUE) { + /* A partial typed tree would change the attributes. Retain the + * label-derived representation when any nested value is absent. */ + cfl_kvlist_destroy(copy); + return 0; + } + if (attributes[index] == NULL || result != 0) { cfl_kvlist_destroy(copy); return -1; } diff --git a/src/cmt_encode_opentelemetry.c b/src/cmt_encode_opentelemetry.c index f384e13c..d3b8fcbc 100644 --- a/src/cmt_encode_opentelemetry.c +++ b/src/cmt_encode_opentelemetry.c @@ -145,7 +145,8 @@ static char *map_type_to_key(int map_type) } } -static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map) +static struct cfl_kvlist *get_legacy_metric_otlp_metadata_context(struct cmt *cmt, + struct cmt_map *map) { struct cfl_kvlist *otlp_root; struct cfl_kvlist *metrics_root; @@ -155,10 +156,6 @@ static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, stru return NULL; } - if (map->otlp_metadata) { - return map->otlp_metadata; - } - otlp_root = fetch_metadata_kvlist_key(cmt->external_metadata, "otlp"); if (otlp_root == NULL) { return NULL; @@ -177,6 +174,15 @@ static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, stru return fetch_metadata_kvlist_key(type_root, map->opts->fqname); } +static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map) +{ + if (map != NULL && map->otlp_metadata) { + return map->otlp_metadata; + } + + return get_legacy_metric_otlp_metadata_context(cmt, map); +} + static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map, struct cmt_metric *sample) @@ -189,7 +195,9 @@ static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, return sample->otlp_metadata; } - metric_context = get_metric_otlp_metadata_context(cmt, map); + /* Native family metadata never carries points. A point an intermediary + * left unbound keeps its legacy entry, as released readers do. */ + metric_context = get_legacy_metric_otlp_metadata_context(cmt, map); if (metric_context == NULL) { return NULL; } diff --git a/src/cmt_mpack_utils.c b/src/cmt_mpack_utils.c index ea67d5ef..ddbc4f9a 100644 --- a/src/cmt_mpack_utils.c +++ b/src/cmt_mpack_utils.c @@ -332,5 +332,11 @@ int cmt_mpack_peek_array_length(mpack_reader_t *reader) return 0; } + /* Callers size allocations from this value before the array is read, so + * a declared count that cmt_mpack_unpack_array() rejects reports zero. */ + if (CMT_MPACK_MAX_ARRAY_ENTRY_COUNT < mpack_tag_array_count(&tag)) { + return 0; + } + return mpack_tag_array_count(&tag); } diff --git a/src/cmt_otlp_metadata.c b/src/cmt_otlp_metadata.c index d0b67041..72487a1c 100644 --- a/src/cmt_otlp_metadata.c +++ b/src/cmt_otlp_metadata.c @@ -26,7 +26,10 @@ #include #include #include +#include +#include +#include #include #include #include @@ -240,13 +243,152 @@ static int pack_optional_map(mpack_writer_t *writer, struct cfl_kvlist *map) return 0; } +static void fingerprint_u64(cfl_hash_state_t *state, uint64_t value) +{ + unsigned char bytes[8]; + int index; + + /* byte order independent: writer and reader may differ */ + for (index = 0; index < 8; index++) { + bytes[index] = (unsigned char) (value >> (index * 8)); + } + cfl_hash_64bits_update(state, bytes, sizeof(bytes)); +} + +static uint64_t double_bits(double value) +{ + uint64_t bits; + + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +/* + * Fingerprint of the complete value a point carries through the MessagePack + * codec, read as pack_metric() reads it, distributions included. Points of + * one series and time window that differ only in what the labels cannot + * hold, such as attribute types, usually differ in value too. + */ +static int point_fingerprint(struct cmt_map *map, struct cmt_metric *sample, + uint64_t *fingerprint) +{ + struct cmt_exp_histogram_snapshot snapshot; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + cfl_hash_state_t state; + int64_t int64_value; + uint64_t uint64_value; + size_t index; + int type; + + cfl_hash_64bits_reset(&state); + fingerprint_u64(&state, map->type); + if (map->type == CMT_HISTOGRAM) { + histogram = (struct cmt_histogram *) map->parent; + for (index = 0; index <= histogram->buckets->count; index++) { + fingerprint_u64(&state, cmt_metric_hist_get_value(sample, index)); + } + fingerprint_u64(&state, cmt_metric_hist_get_count_value(sample)); + fingerprint_u64(&state, double_bits(cmt_metric_hist_get_sum_value(sample))); + } + else if (map->type == CMT_SUMMARY) { + summary = (struct cmt_summary *) map->parent; + fingerprint_u64(&state, cmt_atomic_load(&sample->sum_quantiles_set)); + for (index = 0; sample->sum_quantiles != NULL && + index < summary->quantiles_count; index++) { + fingerprint_u64(&state, cmt_atomic_load(&sample->sum_quantiles[index])); + } + fingerprint_u64(&state, cmt_summary_get_count_value(sample)); + fingerprint_u64(&state, double_bits(cmt_summary_get_sum_value(sample))); + } + else if (map->type == CMT_EXP_HISTOGRAM) { + if (cmt_metric_exp_hist_get_snapshot(sample, &snapshot) != 0) { + return -1; + } + fingerprint_u64(&state, (uint64_t) (int64_t) snapshot.scale); + fingerprint_u64(&state, snapshot.zero_count); + fingerprint_u64(&state, double_bits(snapshot.zero_threshold)); + fingerprint_u64(&state, (uint64_t) (int64_t) snapshot.positive_offset); + fingerprint_u64(&state, snapshot.positive_count); + for (index = 0; index < snapshot.positive_count; index++) { + fingerprint_u64(&state, snapshot.positive_buckets[index]); + } + fingerprint_u64(&state, (uint64_t) (int64_t) snapshot.negative_offset); + fingerprint_u64(&state, snapshot.negative_count); + for (index = 0; index < snapshot.negative_count; index++) { + fingerprint_u64(&state, snapshot.negative_buckets[index]); + } + fingerprint_u64(&state, snapshot.count); + fingerprint_u64(&state, snapshot.sum_set); + fingerprint_u64(&state, snapshot.sum); + cmt_metric_exp_hist_snapshot_destroy(&snapshot); + } + else { + cmt_metric_get_value_snapshot(sample, &type, &int64_value, &uint64_value); + fingerprint_u64(&state, type); + if (type == CMT_METRIC_VALUE_INT64) { + fingerprint_u64(&state, (uint64_t) int64_value); + } + else if (type == CMT_METRIC_VALUE_UINT64) { + fingerprint_u64(&state, uint64_value); + } + else { + fingerprint_u64(&state, double_bits(cmt_metric_get_value(sample))); + } + } + *fingerprint = cfl_hash_64bits_digest(&state); + return 0; +} + +/* + * A point is identified as in cmt_cat() by label values, time and start + * time, plus a fingerprint of its values. The series hash concatenates label + * values without boundaries, so distinct label sets can share it: it only + * accelerates the comparison. Values are written as pack_metric() writes + * labels, so they compare against the labels a reader decodes. The start + * time is appended only when set, so its absence stays distinct. A point + * whose fingerprint cannot be computed records zero and is unlikely to bind. + */ +static void pack_point_id(mpack_writer_t *writer, struct cmt_map *map, + struct cmt_metric *sample) +{ + int has_start; + uint64_t fingerprint; + struct cfl_list *head; + struct cmt_map_label *label; + + if (point_fingerprint(map, sample, &fingerprint) != 0) { + fingerprint = 0; + } + has_start = cmt_metric_has_start_timestamp(sample); + mpack_start_array(writer, has_start ? 5 : 4); + mpack_write_uint(writer, sample->hash); + mpack_write_uint(writer, cmt_metric_get_timestamp(sample)); + mpack_start_array(writer, cfl_list_size(&sample->labels)); + cfl_list_foreach(head, &sample->labels) { + label = cfl_list_entry(head, struct cmt_map_label, _head); + if (label->name == NULL) { + mpack_write_nil(writer); + } + else { + mpack_write_cstr(writer, label->name); + } + } + mpack_finish_array(writer); + mpack_write_uint(writer, fingerprint); + if (has_start) { + mpack_write_uint(writer, cmt_metric_get_start_timestamp(sample)); + } + mpack_finish_array(writer); +} + static int pack_map_metadata(struct cmt_map *map, void *data) { mpack_writer_t *writer = data; struct cfl_list *head; struct cmt_metric *sample; - mpack_start_map(writer, 5); + mpack_start_map(writer, 6); mpack_write_cstr(writer, "type"); mpack_write_int(writer, map->type); mpack_write_cstr(writer, "name"); @@ -270,6 +412,21 @@ static int pack_map_metadata(struct cmt_map *map, void *data) } } mpack_finish_array(writer); + + /* Identity of each entry in "points". An intermediary + * that edits points without updating the sidecar (any released reader) + * changes them, so readers bind by identity instead of by position. + * Readers of v2.3.0 ignore this key. */ + mpack_write_cstr(writer, "point_ids"); + mpack_start_array(writer, cfl_list_size(&map->metrics) + !!map->metric_static_set); + if (map->metric_static_set) { + pack_point_id(writer, map, &map->metric); + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + pack_point_id(writer, map, sample); + } + mpack_finish_array(writer); mpack_finish_map(writer); return 0; } @@ -533,11 +690,20 @@ struct staged_family_metadata { struct cmt_map *map; struct cfl_kvlist *family; struct cfl_kvlist **points; + struct cfl_array *point_ids; + size_t *order; + unsigned char *point_flags; size_t point_count; int allow_reset; int restore_points; }; +/* point_flags bits, per staged point: identity recorded more than once, + * native metadata read back, bound to a decoded sample */ +#define POINT_AMBIGUOUS 1 +#define POINT_HAS_METADATA 2 +#define POINT_MATCHED 4 + struct restore_state { struct cfl_array *families; size_t index; @@ -580,6 +746,268 @@ static int restore_optional_map(struct cfl_kvlist **target, struct cfl_variant * return 0; } +/* Unsigned values are read back as CFL_VARIANT_INT with the same bits. */ +static int is_integer(struct cfl_variant *value) +{ + return value->type == CFL_VARIANT_INT || value->type == CFL_VARIANT_UINT; +} + +static uint64_t integer_bits(struct cfl_variant *value) +{ + if (value->type == CFL_VARIANT_INT) { + return (uint64_t) value->data.as_int64; + } + return value->data.as_uint64; +} + +static int label_values_valid(struct cfl_variant *values) +{ + struct cfl_array *array; + size_t index; + + if (values->type != CFL_VARIANT_ARRAY) { + return CMT_FALSE; + } + array = values->data.as_array; + for (index = 0; index < array->entry_count; index++) { + if (array->entries[index]->type != CFL_VARIANT_STRING && + array->entries[index]->type != CFL_VARIANT_NULL) { + return CMT_FALSE; + } + } + return CMT_TRUE; +} + +/* Each identifier is [series hash, timestamp, label values, fingerprint] + * with the start timestamp appended when set, one per "points" entry. */ +static int point_ids_valid(struct cfl_variant *ids, size_t count) +{ + struct cfl_array *id; + size_t index; + + if (ids->type != CFL_VARIANT_ARRAY || ids->data.as_array->entry_count != count) { + return CMT_FALSE; + } + for (index = 0; index < count; index++) { + if (ids->data.as_array->entries[index]->type != CFL_VARIANT_ARRAY) { + return CMT_FALSE; + } + id = ids->data.as_array->entries[index]->data.as_array; + if (id->entry_count < 4 || id->entry_count > 5 || + !is_integer(id->entries[0]) || !is_integer(id->entries[1]) || + !label_values_valid(id->entries[2]) || !is_integer(id->entries[3]) || + (id->entry_count == 5 && !is_integer(id->entries[4]))) { + return CMT_FALSE; + } + } + return CMT_TRUE; +} + +static int compare_u64(uint64_t left, uint64_t right) +{ + return left < right ? -1 : left > right; +} + +/* Order a label value against one read back from an identifier: nil first, + * then by length and bytes, so values keep their boundaries. */ +static int compare_label_value(struct cfl_variant *value, cfl_sds_t label) +{ + size_t length; + int result; + + if (value->type == CFL_VARIANT_NULL || label == NULL) { + return (value->type != CFL_VARIANT_NULL) - (label != NULL); + } + length = cfl_sds_len(value->data.as_string); + if (length != cfl_sds_len(label)) { + return length < cfl_sds_len(label) ? -1 : 1; + } + result = memcmp(value->data.as_string, label, length); + return (result > 0) - (result < 0); +} + +static int compare_id_header(struct cfl_array *id, uint64_t hash, uint64_t timestamp, + uint64_t fingerprint, int has_start, uint64_t start, + size_t label_count) +{ + int result; + + result = compare_u64(integer_bits(id->entries[0]), hash); + if (result == 0) { + result = compare_u64(integer_bits(id->entries[1]), timestamp); + } + if (result == 0) { + result = compare_u64(integer_bits(id->entries[3]), fingerprint); + } + if (result == 0) { + result = (id->entry_count == 5) - (has_start != 0); + } + if (result == 0 && has_start) { + result = compare_u64(integer_bits(id->entries[4]), start); + } + if (result == 0) { + result = compare_u64(id->entries[2]->data.as_array->entry_count, label_count); + } + return result; +} + +/* Order an identifier against a decoded sample with its fingerprint. */ +static int compare_id_sample(struct cfl_array *id, struct cmt_metric *sample, + uint64_t fingerprint) +{ + struct cfl_array *values; + struct cfl_list *head; + struct cmt_map_label *label; + size_t index = 0; + int has_start; + int result; + + has_start = cmt_metric_has_start_timestamp(sample); + result = compare_id_header(id, sample->hash, cmt_metric_get_timestamp(sample), + fingerprint, has_start, + has_start ? cmt_metric_get_start_timestamp(sample) : 0, + cfl_list_size(&sample->labels)); + values = id->entries[2]->data.as_array; + cfl_list_foreach(head, &sample->labels) { + if (result != 0) { + break; + } + label = cfl_list_entry(head, struct cmt_map_label, _head); + result = compare_label_value(values->entries[index++], label->name); + } + return result; +} + +/* Order two identifiers. */ +static int compare_ids(struct cfl_array *left, struct cfl_array *right) +{ + struct cfl_array *left_values; + struct cfl_array *right_values; + struct cfl_variant *value; + size_t index; + int has_start; + int result; + + has_start = right->entry_count == 5; + right_values = right->entries[2]->data.as_array; + result = compare_id_header(left, integer_bits(right->entries[0]), + integer_bits(right->entries[1]), + integer_bits(right->entries[3]), has_start, + has_start ? integer_bits(right->entries[4]) : 0, + right_values->entry_count); + left_values = left->entries[2]->data.as_array; + for (index = 0; result == 0 && index < left_values->entry_count; index++) { + value = right_values->entries[index]; + result = compare_label_value(left_values->entries[index], + value->type == CFL_VARIANT_NULL ? + NULL : value->data.as_string); + } + return result; +} + +static int compare_staged(struct cfl_array *ids, size_t left, size_t right) +{ + return compare_ids(ids->entries[left]->data.as_array, + ids->entries[right]->data.as_array); +} + +static void sift_down(struct cfl_array *ids, size_t *order, size_t root, size_t count) +{ + size_t child; + size_t swap; + + while ((child = root * 2 + 1) < count) { + if (child + 1 < count && compare_staged(ids, order[child], order[child + 1]) < 0) { + child++; + } + if (compare_staged(ids, order[root], order[child]) >= 0) { + return; + } + swap = order[root]; + order[root] = order[child]; + order[child] = swap; + root = child; + } +} + +/* Sort identifier positions in place. The comparison needs the identifiers, + * which qsort() cannot receive without shared state. */ +static void sort_point_ids(struct cfl_array *ids, size_t *order, size_t count) +{ + size_t index; + size_t swap; + + for (index = count / 2; index-- > 0;) { + sift_down(ids, order, index, count); + } + for (index = count; index-- > 1;) { + swap = order[0]; + order[0] = order[index]; + order[index] = swap; + sift_down(ids, order, 0, index); + } +} + +/* Find the staged entry recorded for this sample, or point_count. */ +static size_t find_point_id(struct staged_family_metadata *staged, struct cmt_metric *sample, + uint64_t fingerprint) +{ + size_t low = 0; + size_t high = staged->point_count; + size_t middle; + + while (low < high) { + middle = low + (high - low) / 2; + if (compare_id_sample(staged->point_ids->entries[staged->order[middle]] + ->data.as_array, sample, fingerprint) < 0) { + low = middle + 1; + } + else { + high = middle; + } + } + if (low >= staged->point_count || + compare_id_sample(staged->point_ids->entries[staged->order[low]]->data.as_array, + sample, fingerprint) != 0) { + return staged->point_count; + } + return staged->order[low]; +} + +/* + * Take the staged metadata recorded for this sample. Without identifiers + * (v2.3.0) points bind by position. Otherwise each sample is looked up by + * identity through the sorted identifiers, wherever it now sits. A sample + * whose identity is not recorded gets no metadata, and neither does one + * whose identity was recorded more than once: nothing tells those points + * apart, so whichever binding was chosen could be another point's. + */ +static struct cfl_kvlist *take_point_metadata(struct staged_family_metadata *staged, + struct cmt_metric *sample, size_t position) +{ + struct cfl_kvlist *metadata; + uint64_t fingerprint; + size_t index; + + if (staged->point_ids == NULL) { + index = position; + } + else { + if (point_fingerprint(staged->map, sample, &fingerprint) != 0) { + return NULL; + } + index = find_point_id(staged, sample, fingerprint); + if (index >= staged->point_count || + (staged->point_flags[index] & POINT_AMBIGUOUS)) { + return NULL; + } + staged->point_flags[index] |= POINT_MATCHED; + } + metadata = staged->points[index]; + staged->points[index] = NULL; + return metadata; +} + static int stage_map_metadata(struct cmt_map *map, void *data) { struct restore_state *state = data; @@ -587,6 +1015,7 @@ static int stage_map_metadata(struct cmt_map *map, void *data) struct cfl_variant *family; struct cfl_variant *points; struct cfl_variant *reset; + struct cfl_variant *ids; size_t index; staged = &state->staged[state->index]; @@ -605,11 +1034,21 @@ static int stage_map_metadata(struct cmt_map *map, void *data) staged->family = NULL; } - /* Label-less entries fold into the static metric on decode, so the point - * list of that family may no longer line up: keep the family metadata - * but skip its point metadata rather than misattributing it. */ - staged->restore_points = points->data.as_array->entry_count == - cfl_list_size(&map->metrics) + !!map->metric_static_set; + ids = cfl_kvlist_fetch(family->data.as_kvlist, "point_ids"); + if (ids != NULL) { + /* Points are bound by labels, time and start time. Identifiers that + * cannot be read back leave the family without point metadata. */ + staged->restore_points = point_ids_valid(ids, points->data.as_array->entry_count); + staged->point_ids = staged->restore_points ? ids->data.as_array : NULL; + } + else { + /* v2.3.0 sidecars carry no identifiers and bind by position. Label-less + * entries fold into the static metric on decode, so the point list of + * that family may no longer line up: keep the family metadata but skip + * its point metadata rather than misattributing it. */ + staged->restore_points = points->data.as_array->entry_count == + cfl_list_size(&map->metrics) + !!map->metric_static_set; + } if (!staged->restore_points) { return 0; } @@ -621,23 +1060,287 @@ static int stage_map_metadata(struct cmt_map *map, void *data) return -1; } } + if (staged->point_ids != NULL && staged->point_count > 0) { + staged->point_flags = calloc(staged->point_count, sizeof(*staged->point_flags)); + if (!staged->point_flags) { + return -1; + } + } for (index = 0; index < staged->point_count; index++) { - /* a point whose metadata cannot be read back keeps none */ + /* a point whose metadata cannot be read back keeps none here, and + * keeps its legacy entry: only metadata read back supersedes it */ if (restore_optional_map(&staged->points[index], points->data.as_array->entries[index]) != 0) { staged->points[index] = NULL; } + if (staged->point_flags != NULL && staged->points[index] != NULL) { + staged->point_flags[index] |= POINT_HAS_METADATA; + } + } + if (staged->point_ids != NULL && staged->point_count > 0) { + staged->order = malloc(staged->point_count * sizeof(*staged->order)); + if (!staged->order) { + return -1; + } + for (index = 0; index < staged->point_count; index++) { + staged->order[index] = index; + } + sort_point_ids(staged->point_ids, staged->order, staged->point_count); + + for (index = 1; index < staged->point_count; index++) { + if (compare_staged(staged->point_ids, staged->order[index - 1], + staged->order[index]) == 0) { + staged->point_flags[staged->order[index - 1]] |= POINT_AMBIGUOUS; + staged->point_flags[staged->order[index]] |= POINT_AMBIGUOUS; + } + } + } + return 0; +} + +static struct cfl_kvlist *legacy_type_root(struct cmt *context, struct cmt_map *map) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(context->external_metadata, "otlp"); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + value = cfl_kvlist_fetch(value->data.as_kvlist, "metrics"); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + value = cfl_kvlist_fetch(value->data.as_kvlist, (char *) legacy_type_key(map->type)); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + return value->data.as_kvlist; +} + +static struct cfl_kvlist *legacy_family(struct cmt *context, struct cmt_map *map) +{ + struct cfl_kvlist *type_root; + struct cfl_variant *value; + + type_root = legacy_type_root(context, map); + if (type_root == NULL) { + return NULL; + } + value = cfl_kvlist_fetch(type_root, map->opts->fqname); + if (value == NULL || value->type != CFL_VARIANT_KVLIST) { + return NULL; + } + return value->data.as_kvlist; +} + +/* Remove legacy containers left empty once native metadata replaced them. */ +static void prune_legacy_family(struct cmt *context, struct cmt_map *map) +{ + struct cfl_kvlist *family; + struct cfl_variant *datapoints; + + family = legacy_family(context, map); + if (family == NULL) { + return; + } + datapoints = cfl_kvlist_fetch(family, "datapoints"); + if (datapoints != NULL && datapoints->type == CFL_VARIANT_KVLIST && + cfl_kvlist_count(datapoints->data.as_kvlist) == 0) { + cfl_kvlist_remove(family, "datapoints"); + } + if (cfl_kvlist_count(family) == 0) { + cfl_kvlist_remove(legacy_type_root(context, map), map->opts->fqname); + } +} + +struct legacy_point_key { + uint64_t hash; + uint64_t timestamp; +}; + +static int compare_legacy_point_keys(const void *left, const void *right) +{ + const struct legacy_point_key *a = left; + const struct legacy_point_key *b = right; + + if (a->hash != b->hash) { + return a->hash < b->hash ? -1 : 1; + } + return (a->timestamp > b->timestamp) - (a->timestamp < b->timestamp); +} + +static void add_sample_key(struct legacy_point_key *keys, size_t *count, + struct cmt_metric *sample) +{ + keys[*count].hash = sample->hash; + keys[*count].timestamp = cmt_metric_get_timestamp(sample); + (*count)++; +} + +/* + * Append the keys two or more decoded samples share. Distinct series can share + * a series hash, and windows of one series a time, so such a legacy entry may + * be any of their points': it is no sample's fallback. Returns -1 when the + * samples cannot be gathered. + */ +static int add_shared_sample_keys(struct legacy_point_key *keys, size_t *count, + struct cmt_map *map) +{ + struct legacy_point_key *samples; + struct cfl_list *head; + size_t total = 0; + size_t index; + + samples = malloc((cfl_list_size(&map->metrics) + 1) * sizeof(*samples)); + if (samples == NULL) { + return -1; + } + if (map->metric_static_set) { + add_sample_key(samples, &total, &map->metric); + } + cfl_list_foreach(head, &map->metrics) { + add_sample_key(samples, &total, cfl_list_entry(head, struct cmt_metric, _head)); + } + qsort(samples, total, sizeof(*samples), compare_legacy_point_keys); + for (index = 1; index < total; index++) { + if (compare_legacy_point_keys(&samples[index - 1], &samples[index]) == 0 && + (*count == 0 || + compare_legacy_point_keys(&keys[*count - 1], &samples[index]) != 0)) { + keys[(*count)++] = samples[index]; + } } + free(samples); return 0; } -static void commit_map_metadata(struct staged_family_metadata *staged) +static void add_restored_key(struct legacy_point_key *keys, size_t *count, + struct cmt_metric *sample) +{ + if (sample->otlp_metadata != NULL) { + keys[*count].hash = sample->hash; + keys[*count].timestamp = cmt_metric_get_timestamp(sample); + (*count)++; + } +} + +/* A legacy "hash:timestamp" key names a restored point when it reads back + * as one of the restored keys in its canonical form, ignoring case as + * cfl_kvlist_remove() did. */ +static int legacy_key_listed(cfl_sds_t name, struct legacy_point_key *keys, size_t count) +{ + struct legacy_point_key key; + char canonical[64]; + char *end; + size_t length; + size_t index; + + key.hash = strtoull(name, &end, 16); + if (*end != ':') { + return CMT_FALSE; + } + key.timestamp = strtoull(end + 1, NULL, 10); + snprintf(canonical, sizeof(canonical), "%" PRIx64 ":%" PRIu64, + key.hash, key.timestamp); + length = strlen(canonical); + if (cfl_sds_len(name) != length) { + return CMT_FALSE; + } + for (index = 0; index < length; index++) { + if (tolower((unsigned char) name[index]) != canonical[index]) { + return CMT_FALSE; + } + } + return bsearch(&key, keys, count, sizeof(*keys), compare_legacy_point_keys) != NULL; +} + +/* + * Drop legacy copies the native metadata supersedes, in one pass over the + * legacy entries: removing them one by one would rescan the list for every + * point. + * + * With identifiers, a legacy entry, keyed by series hash and time only, stays + * only when every point recorded under its key was bound to a decoded sample + * by a unique identity and has no native metadata to read back: the entry is + * then that sample's own, as in released readers. It is dropped when native + * metadata replaced it, and when a recorded point was not bound (removed, + * changed by an intermediary, or ambiguous): the entry describes another or + * an older point. Legacy entries of points never recorded stay. + * Without identifiers (v2.3.0), only the copies of restored points are + * dropped, so points an intermediary left unbound keep what released readers + * keep. + */ +static void drop_legacy_points(struct cfl_kvlist *family, + struct staged_family_metadata *staged) { struct cmt_map *map = staged->map; + struct cfl_array *id; + size_t index; + struct legacy_point_key *keys; + struct cfl_variant *datapoints; + struct cfl_kvpair *pair; + struct cfl_list *head; + struct cfl_list *tmp; + struct cmt_metric *sample; + size_t count = 0; + + if (family == NULL) { + return; + } + datapoints = cfl_kvlist_fetch(family, "datapoints"); + if (datapoints == NULL || datapoints->type != CFL_VARIANT_KVLIST) { + return; + } + keys = malloc(((staged->point_ids != NULL ? staged->point_count : 0) + + cfl_list_size(&map->metrics) + 1) * sizeof(*keys)); + if (keys == NULL) { + /* no entry can be shown to be a safe fallback */ + cfl_kvlist_remove(family, "datapoints"); + return; + } + if (staged->point_ids != NULL) { + for (index = 0; index < staged->point_count; index++) { + if ((staged->point_flags[index] & POINT_MATCHED) && + !(staged->point_flags[index] & POINT_HAS_METADATA)) { + continue; + } + id = staged->point_ids->entries[index]->data.as_array; + keys[count].hash = integer_bits(id->entries[0]); + keys[count].timestamp = integer_bits(id->entries[1]); + count++; + } + } + else { + if (map->metric_static_set) { + add_restored_key(keys, &count, &map->metric); + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + add_restored_key(keys, &count, sample); + } + } + qsort(keys, count, sizeof(*keys), compare_legacy_point_keys); + cfl_list_foreach_safe(head, tmp, &datapoints->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (count > 0 && legacy_key_listed(pair->key, keys, count)) { + cfl_kvpair_destroy(pair); + } + } + free(keys); +} + +static void commit_map_metadata(struct cmt *context, + struct staged_family_metadata *staged) +{ + struct cmt_map *map = staged->map; + struct cfl_kvlist *legacy; struct cfl_list *head; struct cmt_metric *sample; size_t index = 0; + legacy = legacy_family(context, map); + if (legacy != NULL && staged->family != NULL) { + cfl_kvlist_remove(legacy, "metadata"); + } cfl_kvlist_destroy(map->otlp_metadata); map->otlp_metadata = staged->family; staged->family = NULL; @@ -645,19 +1348,20 @@ static void commit_map_metadata(struct staged_family_metadata *staged) ((struct cmt_counter *) map->parent)->allow_reset = staged->allow_reset; } if (!staged->restore_points) { + prune_legacy_family(context, map); return; } if (map->metric_static_set) { cfl_kvlist_destroy(map->metric.otlp_metadata); - map->metric.otlp_metadata = staged->points[index]; - staged->points[index++] = NULL; + map->metric.otlp_metadata = take_point_metadata(staged, &map->metric, index++); } cfl_list_foreach(head, &map->metrics) { sample = cfl_list_entry(head, struct cmt_metric, _head); cfl_kvlist_destroy(sample->otlp_metadata); - sample->otlp_metadata = staged->points[index]; - staged->points[index++] = NULL; + sample->otlp_metadata = take_point_metadata(staged, sample, index++); } + drop_legacy_points(legacy, staged); + prune_legacy_family(context, map); } static int check_map_metadata_shape(struct cmt_map *map, void *data) @@ -694,28 +1398,61 @@ static int check_map_metadata_shape(struct cmt_map *map, void *data) return 0; } -static void drop_legacy_family(struct cmt *context, struct cmt_map *map) +/* + * Drop legacy entries whose key two or more decoded samples share. Distinct + * series can share a series hash, and windows of one series a time, so such + * an entry may be any of their points': it is no sample's fallback. This + * holds whether native metadata was restored, skipped or absent, so it runs + * for every family. Entries one sample owns stay. + */ +static int drop_shared_legacy_keys(struct cmt_map *map, void *data) { - struct cfl_variant *otlp; - struct cfl_variant *metrics; - struct cfl_variant *type_root; + struct cmt *context = data; + struct legacy_point_key *keys; + struct cfl_kvlist *family; + struct cfl_variant *datapoints; + struct cfl_kvpair *pair; + struct cfl_list *head; + struct cfl_list *tmp; + size_t count = 0; - otlp = cfl_kvlist_fetch(context->external_metadata, "otlp"); - if (otlp == NULL || otlp->type != CFL_VARIANT_KVLIST) { - return; + family = legacy_family(context, map); + if (family == NULL) { + return 0; } - metrics = cfl_kvlist_fetch(otlp->data.as_kvlist, "metrics"); - if (metrics == NULL || metrics->type != CFL_VARIANT_KVLIST) { - return; + datapoints = cfl_kvlist_fetch(family, "datapoints"); + if (datapoints == NULL || datapoints->type != CFL_VARIANT_KVLIST) { + return 0; } - type_root = cfl_kvlist_fetch(metrics->data.as_kvlist, (char *) legacy_type_key(map->type)); - if (type_root == NULL || type_root->type != CFL_VARIANT_KVLIST) { - return; + keys = malloc((cfl_list_size(&map->metrics) + 1) * sizeof(*keys)); + if (keys == NULL || add_shared_sample_keys(keys, &count, map) != 0) { + /* no entry can be shown to be a safe fallback */ + cfl_kvlist_remove(family, "datapoints"); } - cfl_kvlist_remove(type_root->data.as_kvlist, map->opts->fqname); + else if (count > 0) { + cfl_list_foreach_safe(head, tmp, &datapoints->data.as_kvlist->list) { + pair = cfl_list_entry(head, struct cfl_kvpair, _head); + if (legacy_key_listed(pair->key, keys, count)) { + cfl_kvpair_destroy(pair); + } + } + } + free(keys); + prune_legacy_family(context, map); + return 0; } +static void restore_native_metadata(struct cmt *context); + int cmt_otlp_metadata_restore(struct cmt *context) +{ + restore_native_metadata(context); + visit_maps(context, drop_shared_legacy_keys, context); + /* Malformed optional metadata never invalidates the native metric payload. */ + return 0; +} + +static void restore_native_metadata(struct cmt *context) { struct cfl_variant *value; struct restore_state state; @@ -725,11 +1462,11 @@ int cmt_otlp_metadata_restore(struct cmt *context) value = cfl_kvlist_fetch(context->internal_metadata, METADATA_KEY); if (value == NULL) { - return 0; + return; } if (value->type != CFL_VARIANT_ARRAY) { cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); - return 0; + return; } state.families = value->data.as_array; state.index = 0; @@ -737,20 +1474,18 @@ int cmt_otlp_metadata_restore(struct cmt *context) if (result != 0 || state.index != state.families->entry_count) { /* Optional metadata must not invalidate an otherwise usable chunk. */ cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); - return 0; + return; } state.staged = calloc(state.families->entry_count, sizeof(*state.staged)); if (!state.staged && state.families->entry_count > 0) { cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); - return 0; + return; } state.index = 0; result = visit_maps(context, stage_map_metadata, &state); if (result == 0) { for (family_index = 0; family_index < state.families->entry_count; family_index++) { - /* the legacy copy written for older readers is superseded */ - drop_legacy_family(context, state.staged[family_index].map); - commit_map_metadata(&state.staged[family_index]); + commit_map_metadata(context, &state.staged[family_index]); } } for (family_index = 0; family_index < state.families->entry_count; family_index++) { @@ -761,11 +1496,11 @@ int cmt_otlp_metadata_restore(struct cmt *context) } } free(state.staged[family_index].points); + free(state.staged[family_index].order); + free(state.staged[family_index].point_flags); } free(state.staged); cfl_kvlist_remove(context->internal_metadata, METADATA_KEY); - /* Malformed optional metadata never invalidates the native metric payload. */ - return 0; } int cmt_otlp_capture_labels(struct cmt_map *map, struct cmt_metric *sample) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5b9c9bc3..06555709 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -20,6 +20,8 @@ set(UNIT_TESTS_FILES msgpack_temporality.c format_conversion.c expire.c + otlp_pipeline.c + msgpack_compat.c ) if (CMT_BUILD_PROMETHEUS_TEXT_DECODER) diff --git a/tests/cat.c b/tests/cat.c index 753dbd35..80d48322 100644 --- a/tests/cat.c +++ b/tests/cat.c @@ -33,6 +33,7 @@ #include #include #include +#include #include "cmt_tests.h" @@ -1265,6 +1266,60 @@ static uint64_t sample_flags(struct cmt_metric *sample) return UINT64_MAX; } +void test_cat_clears_stale_otlp_point_metadata() +{ + int label_count; + char *keys[] = {"key"}; + char *values[] = {"v"}; + struct cmt *dst; + struct cmt *src; + struct cmt_gauge *dst_gauge; + struct cmt_gauge *src_gauge; + struct cmt_metric *dst_sample; + struct cmt_metric *src_sample; + + /* Exercise both the static and label-indexed sample replacement paths. */ + for (label_count = 0; label_count <= 1; label_count++) { + dst = cmt_create(); + src = cmt_create(); + TEST_ASSERT(dst != NULL && src != NULL); + + dst_gauge = cmt_gauge_create(dst, "", "", "otlp_gauge", "otlp gauge", + label_count, keys); + src_gauge = cmt_gauge_create(src, "", "", "otlp_gauge", "otlp gauge", + label_count, keys); + TEST_ASSERT(dst_gauge != NULL && src_gauge != NULL); + TEST_ASSERT(cmt_gauge_set(dst_gauge, 100, 1.0, label_count, values) == 0); + TEST_ASSERT(cmt_gauge_set(src_gauge, 200, 2.5, label_count, values) == 0); + + dst_sample = cmt_map_metric_get(&dst_gauge->opts, dst_gauge->map, + label_count, values, CMT_FALSE); + src_sample = cmt_map_metric_get(&src_gauge->opts, src_gauge->map, + label_count, values, CMT_FALSE); + TEST_ASSERT(dst_sample != NULL && src_sample != NULL); + dst_sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(dst_sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(dst_sample->otlp_metadata, + "flags", 1) == 0); + cmt_metric_set_start_timestamp(dst_sample, 50); + TEST_ASSERT(src_sample->otlp_metadata == NULL); + + TEST_ASSERT(cmt_cat(dst, src) == 0); + TEST_CHECK(dst_sample->otlp_metadata == NULL); + TEST_CHECK(cmt_metric_get_value(dst_sample) == 2.5); + TEST_CHECK(cmt_metric_get_timestamp(dst_sample) == 200); + TEST_CHECK(!cmt_metric_has_start_timestamp(dst_sample)); + TEST_CHECK(cfl_list_size(&dst_gauge->map->metrics) == label_count); + + /* Repeated copies must also tolerate an already empty destination. */ + TEST_ASSERT(cmt_cat(dst, src) == 0); + TEST_CHECK(dst_sample->otlp_metadata == NULL); + + cmt_destroy(src); + cmt_destroy(dst); + } +} + /* Samples carrying OTLP metadata must merge by labels, never duplicate. */ void test_cat_otlp_metadata_samples_merge_by_labels() { @@ -1409,7 +1464,208 @@ void test_cat_otlp_metadata_distinct_points_kept() cmt_destroy(final); } +/* A family with more label keys than the destination cannot be merged: the + * destination must stay encodable and keep its samples. */ +void test_cat_rejects_larger_label_schema() +{ + struct cmt *a; + struct cmt *b; + struct cmt *dst; + struct cmt_gauge *gauge_a; + struct cmt_gauge *gauge_b; + struct cmt_gauge *out; + cfl_sds_t wire; + + a = cmt_create(); + b = cmt_create(); + dst = cmt_create(); + TEST_ASSERT(a != NULL && b != NULL && dst != NULL); + gauge_a = cmt_gauge_create(a, "", "", "g", "g", 1, (char *[]) {"a"}); + gauge_b = cmt_gauge_create(b, "", "", "g", "g", 2, (char *[]) {"a", "b"}); + TEST_ASSERT(gauge_a != NULL && gauge_b != NULL); + TEST_ASSERT(cmt_gauge_set(gauge_a, 100, 1, 1, (char *[]) {"x"}) == 0); + TEST_ASSERT(cmt_gauge_set(gauge_b, 200, 2, 2, (char *[]) {"x", "y"}) == 0); + + TEST_CHECK(cmt_cat(dst, a) == 0); + TEST_CHECK(cmt_cat(dst, b) == -1); + + out = cfl_list_entry_first(&dst->gauges, struct cmt_gauge, _head); + TEST_CHECK(out->map->label_count == 1); + TEST_CHECK(cfl_list_size(&out->map->metrics) == 1); + wire = cmt_encode_opentelemetry_create(dst); + TEST_CHECK(wire != NULL); + if (wire != NULL) { + cmt_encode_opentelemetry_destroy(wire); + } + + cmt_destroy(a); + cmt_destroy(b); + cmt_destroy(dst); +} + +static struct cmt *semantics_counter(int aggregation_type, int allow_reset, int with_sample, + char *unit) +{ + struct cmt *ctx; + struct cmt_counter *counter; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + counter = cmt_counter_create(ctx, "", "", "requests", "requests", 1, (char *[]) {"site"}); + TEST_ASSERT(counter != NULL); + counter->aggregation_type = aggregation_type; + counter->allow_reset = allow_reset; + if (unit != NULL) { + counter->map->unit = cfl_sds_create(unit); + TEST_ASSERT(counter->map->unit != NULL); + } + if (with_sample) { + TEST_ASSERT(cmt_counter_set(counter, 100, 5, 1, (char *[]) {"b"}) == 0); + } + return ctx; +} + +static struct cmt *semantics_histogram(int aggregation_type) +{ + struct cmt *ctx; + struct cmt_histogram *histogram; + uint64_t buckets[] = {1, 2}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + histogram = cmt_histogram_create(ctx, "", "", "latency", "latency", + cmt_histogram_buckets_create(1, 1.0), 0, NULL); + TEST_ASSERT(histogram != NULL); + histogram->aggregation_type = aggregation_type; + TEST_ASSERT(cmt_histogram_set_default(histogram, 100, buckets, 1.5, 2, 0, NULL) == 0); + return ctx; +} + +/* Later contributors must not change how samples already in an existing + * family are read: an empty one is a no-op and a conflicting one fails + * without changes. */ +void test_cat_keeps_existing_family_semantics() +{ + struct cmt *dst; + struct cmt *src; + struct cmt_counter *counter; + struct cmt_histogram *histogram; + + dst = cmt_create(); + TEST_ASSERT(dst != NULL); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, NULL); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + counter = cfl_list_entry_first(&dst->counters, struct cmt_counter, _head); + + /* empty contributors: delta, non-monotonic, with a unit */ + src = semantics_counter(CMT_AGGREGATION_TYPE_DELTA, CMT_TRUE, CMT_FALSE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + TEST_CHECK(counter->allow_reset == CMT_FALSE); + TEST_CHECK(counter->map->unit == NULL); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + + /* contributors with samples and conflicting semantics */ + src = semantics_counter(CMT_AGGREGATION_TYPE_DELTA, CMT_FALSE, CMT_TRUE, NULL); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_TRUE, CMT_TRUE, NULL); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + TEST_CHECK(counter->allow_reset == CMT_FALSE); + TEST_CHECK(cmt_metric_get_value(cfl_list_entry_first(&counter->map->metrics, + struct cmt_metric, _head)) == 5); + cmt_destroy(dst); + + /* unspecified temporality (Fluent Bit 5.0.x histograms) merges with + * either and keeps the destination's */ + dst = semantics_histogram(CMT_AGGREGATION_TYPE_CUMULATIVE); + src = semantics_histogram(CMT_AGGREGATION_TYPE_UNSPECIFIED); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + histogram = cfl_list_entry_first(&dst->histograms, struct cmt_histogram, _head); + TEST_CHECK(histogram->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + src = semantics_histogram(CMT_AGGREGATION_TYPE_DELTA); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + TEST_CHECK(histogram->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE); + cmt_destroy(dst); +} + +/* A destination family that holds no samples has nothing to reinterpret: + * it takes the semantics of the first contributor that brings samples. */ +void test_cat_empty_family_adopts_semantics() +{ + struct cmt *dst; + struct cmt *src; + struct cmt_counter *counter; + + dst = cmt_create(); + TEST_ASSERT(dst != NULL); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_FALSE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + counter = cfl_list_entry_first(&dst->counters, struct cmt_counter, _head); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 0); + + src = semantics_counter(CMT_AGGREGATION_TYPE_DELTA, CMT_TRUE, CMT_TRUE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); + TEST_CHECK(counter->allow_reset == CMT_TRUE); + TEST_CHECK(counter->map->unit != NULL && strcmp(counter->map->unit, "ms") == 0); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + + /* once it holds samples, its semantics stay */ + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + TEST_CHECK(counter->aggregation_type == CMT_AGGREGATION_TYPE_DELTA); + cmt_destroy(dst); +} + +/* Values are never converted, so families with different explicit units + * cannot be merged. An empty unit is unspecified. */ +void test_cat_rejects_conflicting_units() +{ + struct cmt *dst; + struct cmt *src; + struct cmt_counter *counter; + + dst = cmt_create(); + TEST_ASSERT(dst != NULL); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + counter = cfl_list_entry_first(&dst->counters, struct cmt_counter, _head); + + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == -1); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_FALSE, "ms"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, ""); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + src = semantics_counter(CMT_AGGREGATION_TYPE_CUMULATIVE, CMT_FALSE, CMT_TRUE, "s"); + TEST_CHECK(cmt_cat(dst, src) == 0); + cmt_destroy(src); + + TEST_CHECK(strcmp(counter->map->unit, "s") == 0); + TEST_CHECK(cfl_list_size(&counter->map->metrics) == 1); + cmt_destroy(dst); +} + TEST_LIST = { + {"cat_rejects_larger_label_schema", test_cat_rejects_larger_label_schema}, + {"cat_keeps_existing_family_semantics", test_cat_keeps_existing_family_semantics}, + {"cat_rejects_conflicting_units", test_cat_rejects_conflicting_units}, + {"cat_empty_family_adopts_semantics", test_cat_empty_family_adopts_semantics}, + {"cat_clears_stale_otlp_point_metadata", test_cat_clears_stale_otlp_point_metadata}, {"cat_otlp_metadata_distinct_points_kept", test_cat_otlp_metadata_distinct_points_kept}, {"cat_otlp_metadata_samples_merge_by_labels", test_cat_otlp_metadata_samples_merge_by_labels}, {"cat_many_contexts", test_cat_many_contexts}, diff --git a/tests/data/all-types-release.c b/tests/data/all-types-release.c new file mode 100644 index 00000000..fbf53d38 --- /dev/null +++ b/tests/data/all-types-release.c @@ -0,0 +1,103 @@ +/* + * Generator for all-types-v5.0.0.msgpack, all-types-v5.0.11.msgpack and + * all-types-v5.1.5.msgpack: one context with every metric type, written by the + * CMetrics library bundled with that Fluent Bit release (2.0.5, 2.1.5 and + * 2.2.5). These are the payloads those agents send over forward and keep in + * filesystem chunks, so current readers must keep decoding them. + * + * Build it against the headers and static libraries of lib/cmetrics, lib/cfl + * and lib/fluent-otel-proto from the Fluent Bit release tag and redirect stdout + * to the fixture. The fixtures are versioned on purpose: tests must never + * regenerate them with the current encoder. tests/msgpack_compat.c asserts + * the values below. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(void) +{ + struct cmt *ctx = cmt_create(); + struct cmt_counter *counter; + struct cmt_counter *delta; + struct cmt_gauge *gauge; + struct cmt_gauge *signed_gauge; + struct cmt_gauge *unsigned_gauge; + struct cmt_untyped *untyped; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *metric; + double quantiles[] = {0.5, 0.9}; + double quantile_values[] = {0.2, 0.8}; + uint64_t buckets[] = {1, 3, 4, 5}; + uint64_t positive[] = {1, 2}; + uint64_t negative[] = {3}; + char *buffer; + size_t size; + + cmt_label_add(ctx, "cluster", "prod"); + + counter = cmt_counter_create(ctx, "fixture", "http", "requests_total", "requests", + 1, (char *[]) {"method"}); + cmt_counter_set(counter, 1000, 10, 1, (char *[]) {"GET"}); + cmt_counter_set(counter, 1000, 4, 1, (char *[]) {"POST"}); + metric = cmt_map_metric_get(&counter->opts, counter->map, 1, (char *[]) {"GET"}, 0); + cmt_metric_set_start_timestamp(metric, 500); + + delta = cmt_counter_create(ctx, "", "", "fixture_delta_total", "delta", 0, NULL); + delta->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + cmt_counter_allow_reset(delta); + cmt_counter_set(delta, 1000, 3, 0, NULL); + + gauge = cmt_gauge_create(ctx, "", "", "fixture_temperature", "temperature", + 1, (char *[]) {"room"}); + cmt_gauge_set(gauge, 1000, -12.5, 1, (char *[]) {"lab"}); + cmt_gauge_set(gauge, 1000, 21, 1, (char *[]) {""}); + + signed_gauge = cmt_gauge_create(ctx, "", "", "fixture_signed", "signed", 0, NULL); + cmt_gauge_set(signed_gauge, 1000, 0, 0, NULL); + cmt_metric_set_int64(&signed_gauge->map->metric, 1000, -9007199254740993LL); + + unsigned_gauge = cmt_gauge_create(ctx, "", "", "fixture_unsigned", "unsigned", 0, NULL); + cmt_gauge_set(unsigned_gauge, 1000, 0, 0, NULL); + cmt_metric_set_uint64(&unsigned_gauge->map->metric, 1000, UINT64_MAX); + + untyped = cmt_untyped_create(ctx, "", "", "fixture_untyped", "untyped", + 1, (char *[]) {"k"}); + cmt_untyped_set(untyped, 1000, 1.5, 1, (char *[]) {"v"}); + + histogram = cmt_histogram_create(ctx, "", "", "fixture_latency_seconds", "latency", + cmt_histogram_buckets_create(3, 0.1, 1.0, 10.0), + 1, (char *[]) {"route"}); + histogram->aggregation_type = CMT_AGGREGATION_TYPE_CUMULATIVE; + histogram->map->unit = cfl_sds_create("s"); + cmt_histogram_set_default(histogram, 1000, buckets, 6.5, 5, 1, (char *[]) {"/"}); + + summary = cmt_summary_create(ctx, "", "", "fixture_rpc_seconds", "rpc", 2, quantiles, + 1, (char *[]) {"svc"}); + cmt_summary_set_default(summary, 1000, quantile_values, 10, 20, 1, (char *[]) {"api"}); + + exp_histogram = cmt_exp_histogram_create(ctx, "", "", "fixture_exp", "exp", + 1, (char *[]) {"op"}); + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + cmt_exp_histogram_set_default(exp_histogram, 1000, 2, 1, 0.001, -1, 2, positive, + 0, 1, negative, 1, 4.5, 7, 1, (char *[]) {"read"}); + + if (cmt_encode_msgpack_create(ctx, &buffer, &size) != 0) { + return 1; + } + fwrite(buffer, 1, size, stdout); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); + return 0; +} diff --git a/tests/data/all-types-v5.0.0.msgpack b/tests/data/all-types-v5.0.0.msgpack new file mode 100644 index 00000000..52f1f322 Binary files /dev/null and b/tests/data/all-types-v5.0.0.msgpack differ diff --git a/tests/data/all-types-v5.0.11.msgpack b/tests/data/all-types-v5.0.11.msgpack new file mode 100644 index 00000000..20fe5c0e Binary files /dev/null and b/tests/data/all-types-v5.0.11.msgpack differ diff --git a/tests/data/all-types-v5.1.5.msgpack b/tests/data/all-types-v5.1.5.msgpack new file mode 100644 index 00000000..b850b0d4 Binary files /dev/null and b/tests/data/all-types-v5.1.5.msgpack differ diff --git a/tests/msgpack_abi.c b/tests/msgpack_abi.c index 2be211eb..44a903dd 100644 --- a/tests/msgpack_abi.c +++ b/tests/msgpack_abi.c @@ -420,8 +420,8 @@ static void test_point_metadata_and_unlabeled_windows(void) /* A further label-less point is the same series: like released readers, * it replaces the static metric (last one wins) instead of creating a - * duplicate series. Its point metadata no longer lines up with the - * sidecar and is skipped, while the values and the chunk survive. */ + * duplicate series. The sidecar binds point metadata by point identity, + * so the surviving point keeps its own metadata. */ source = create_static_gauge_with_metadata(); TEST_ASSERT(source != NULL); gauge = cfl_list_entry_first(&source->gauges, struct cmt_gauge, _head); @@ -443,7 +443,10 @@ static void test_point_metadata_and_unlabeled_windows(void) TEST_CHECK(cfl_list_size(&result->map->metrics) == 0); TEST_CHECK(cmt_metric_get_timestamp(&result->map->metric) == 2); TEST_CHECK(cmt_metric_get_value(&result->map->metric) == 2.0); - TEST_CHECK(result->map->metric.otlp_metadata == NULL); + value = cfl_kvlist_fetch(result->map->metric.otlp_metadata, "flags"); + TEST_ASSERT(value != NULL); + TEST_CHECK((value->type == CFL_VARIANT_UINT && value->data.as_uint64 == 3) || + (value->type == CFL_VARIANT_INT && value->data.as_int64 == 3)); cmt_decode_msgpack_destroy(decoded); cmt_encode_msgpack_destroy(buffer); cmt_destroy(source); diff --git a/tests/msgpack_compat.c b/tests/msgpack_compat.c new file mode 100644 index 00000000..56c4ecb8 --- /dev/null +++ b/tests/msgpack_compat.c @@ -0,0 +1,826 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Compatibility of the MessagePack format between Fluent Bit releases. Agents + * exchange it over forward and keep it in filesystem chunks across upgrades + * and rollbacks, and released readers reject any key they do not know. + * + * Fixtures under tests/data/all-types-*.msgpack were written by the CMetrics + * bundled with each Fluent Bit release (see tests/data/all-types-release.c). + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "cmt_tests.h" + +static const char *release_fixtures[] = { + "all-types-v5.0.0.msgpack", + "all-types-v5.0.11.msgpack", + "all-types-v5.1.5.msgpack", +}; + +/* Fluent Bit v5.0.0 (CMetrics 2.0.5) does not know "unit", and Fluent Bit 5.0 + * releases write the aggregation type of counters only: their histograms and + * exponential histograms arrive without temporality. */ +#define RELEASE_FIXTURE_HAS_UNIT(index) ((index) > 0) +#define RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index) ((index) > 1) + +static cfl_sds_t read_fixture(const char *name) +{ + char path[1024]; + + snprintf(path, sizeof(path), "%s/%s", CMT_TESTS_DATA_PATH, name); + return read_file(path); +} + +static struct cmt_map *find_map(struct cmt *ctx, int type, const char *name) +{ + struct cfl_list *list; + struct cfl_list *head; + struct cmt_map *map; + + switch (type) { + case CMT_COUNTER: + list = &ctx->counters; + break; + case CMT_GAUGE: + list = &ctx->gauges; + break; + case CMT_UNTYPED: + list = &ctx->untypeds; + break; + case CMT_SUMMARY: + list = &ctx->summaries; + break; + case CMT_HISTOGRAM: + list = &ctx->histograms; + break; + default: + list = &ctx->exp_histograms; + break; + } + + cfl_list_foreach(head, list) { + /* Every family struct starts with its opts and map pointers in the + * same order after a type-specific header; use the typed entries. */ + switch (type) { + case CMT_COUNTER: + map = cfl_list_entry(head, struct cmt_counter, _head)->map; + break; + case CMT_GAUGE: + map = cfl_list_entry(head, struct cmt_gauge, _head)->map; + break; + case CMT_UNTYPED: + map = cfl_list_entry(head, struct cmt_untyped, _head)->map; + break; + case CMT_SUMMARY: + map = cfl_list_entry(head, struct cmt_summary, _head)->map; + break; + case CMT_HISTOGRAM: + map = cfl_list_entry(head, struct cmt_histogram, _head)->map; + break; + default: + map = cfl_list_entry(head, struct cmt_exp_histogram, _head)->map; + break; + } + if (strcmp(map->opts->fqname, name) == 0) { + return map; + } + } + return NULL; +} + +static struct cmt_metric *find_sample(struct cmt_map *map, const char *label_value) +{ + struct cfl_list *head; + struct cmt_metric *sample; + struct cmt_map_label *label; + + if (label_value == NULL) { + return map->metric_static_set ? &map->metric : NULL; + } + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + label = cfl_list_entry_first(&sample->labels, struct cmt_map_label, _head); + if (label != NULL && label->name != NULL && strcmp(label->name, label_value) == 0) { + return sample; + } + } + return NULL; +} + +static int has_static_label(struct cmt *ctx, const char *key, const char *value) +{ + struct cfl_list *head; + struct cmt_label *label; + + cfl_list_foreach(head, &ctx->static_labels->list) { + label = cfl_list_entry(head, struct cmt_label, _head); + if (strcmp(label->key, key) == 0 && strcmp(label->val, value) == 0) { + return CMT_TRUE; + } + } + return CMT_FALSE; +} + +/* The content written by tests/data/all-types-release.c. */ +static void check_release_content(struct cmt *ctx, int has_unit, int has_temporality, + const char *source) +{ + struct cmt_map *map; + struct cmt_metric *sample; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_exp_histogram_snapshot snapshot; + + TEST_CASE(source); + TEST_CHECK(has_static_label(ctx, "cluster", "prod")); + + map = find_map(ctx, CMT_COUNTER, "fixture_http_requests_total"); + TEST_ASSERT(map != NULL); + TEST_CHECK(map->label_count == 1); + sample = find_sample(map, "GET"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 10); + TEST_CHECK(cmt_metric_get_timestamp(sample) == 1000); + TEST_CHECK(cmt_metric_has_start_timestamp(sample)); + TEST_CHECK(cmt_metric_get_start_timestamp(sample) == 500); + sample = find_sample(map, "POST"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 4); + TEST_CHECK(!cmt_metric_has_start_timestamp(sample)); + + map = find_map(ctx, CMT_COUNTER, "fixture_delta_total"); + TEST_ASSERT(map != NULL); + TEST_CHECK(((struct cmt_counter *) map->parent)->aggregation_type == + CMT_AGGREGATION_TYPE_DELTA); + TEST_CHECK(cmt_metric_get_value(&map->metric) == 3); + + map = find_map(ctx, CMT_GAUGE, "fixture_temperature"); + TEST_ASSERT(map != NULL); + sample = find_sample(map, "lab"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == -12.5); + sample = find_sample(map, ""); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 21); + + map = find_map(ctx, CMT_GAUGE, "fixture_signed"); + TEST_ASSERT(map != NULL); + TEST_CHECK(cmt_metric_get_value_type(&map->metric) == CMT_METRIC_VALUE_INT64); + TEST_CHECK(cmt_metric_get_int64_value(&map->metric) == -9007199254740993LL); + + map = find_map(ctx, CMT_GAUGE, "fixture_unsigned"); + TEST_ASSERT(map != NULL); + TEST_CHECK(cmt_metric_get_value_type(&map->metric) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK(cmt_metric_get_uint64_value(&map->metric) == UINT64_MAX); + + map = find_map(ctx, CMT_UNTYPED, "fixture_untyped"); + TEST_ASSERT(map != NULL); + sample = find_sample(map, "v"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value(sample) == 1.5); + + map = find_map(ctx, CMT_HISTOGRAM, "fixture_latency_seconds"); + TEST_ASSERT(map != NULL); + histogram = map->parent; + TEST_CHECK(histogram->aggregation_type == (has_temporality ? + CMT_AGGREGATION_TYPE_CUMULATIVE : + CMT_AGGREGATION_TYPE_UNSPECIFIED)); + TEST_ASSERT(histogram->buckets != NULL && histogram->buckets->count == 3); + TEST_CHECK(histogram->buckets->upper_bounds[0] == 0.1); + TEST_CHECK(histogram->buckets->upper_bounds[2] == 10.0); + if (has_unit) { + TEST_CHECK(map->unit != NULL && strcmp(map->unit, "s") == 0); + } + sample = find_sample(map, "/"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_hist_get_value(sample, 0) == 1); + TEST_CHECK(cmt_metric_hist_get_value(sample, 1) == 3); + TEST_CHECK(cmt_metric_hist_get_value(sample, 2) == 4); + TEST_CHECK(cmt_metric_hist_get_value(sample, 3) == 5); + TEST_CHECK(cmt_metric_hist_get_count_value(sample) == 5); + TEST_CHECK(cmt_metric_hist_get_sum_value(sample) == 6.5); + + map = find_map(ctx, CMT_SUMMARY, "fixture_rpc_seconds"); + TEST_ASSERT(map != NULL); + summary = map->parent; + TEST_ASSERT(summary->quantiles_count == 2); + TEST_CHECK(summary->quantiles[0] == 0.5 && summary->quantiles[1] == 0.9); + sample = find_sample(map, "api"); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_summary_quantile_get_value(sample, 0) == 0.2); + TEST_CHECK(cmt_summary_quantile_get_value(sample, 1) == 0.8); + TEST_CHECK(cmt_summary_get_sum_value(sample) == 10); + TEST_CHECK(cmt_summary_get_count_value(sample) == 20); + + map = find_map(ctx, CMT_EXP_HISTOGRAM, "fixture_exp"); + TEST_ASSERT(map != NULL); + exp_histogram = map->parent; + TEST_CHECK(exp_histogram->aggregation_type == (has_temporality ? + CMT_AGGREGATION_TYPE_DELTA : + CMT_AGGREGATION_TYPE_UNSPECIFIED)); + sample = find_sample(map, "read"); + TEST_ASSERT(sample != NULL); + TEST_ASSERT(cmt_metric_exp_hist_get_snapshot(sample, &snapshot) == 0); + TEST_CHECK(snapshot.scale == 2); + TEST_CHECK(snapshot.zero_count == 1); + TEST_CHECK(snapshot.zero_threshold == 0.001); + TEST_CHECK(snapshot.positive_offset == -1 && snapshot.positive_count == 2); + TEST_CHECK(snapshot.positive_count == 2 && + snapshot.positive_buckets[0] == 1 && snapshot.positive_buckets[1] == 2); + TEST_CHECK(snapshot.negative_offset == 0 && snapshot.negative_count == 1); + TEST_CHECK(snapshot.negative_count == 1 && snapshot.negative_buckets[0] == 3); + TEST_CHECK(snapshot.count == 7); + TEST_CHECK(snapshot.sum_set); + cmt_metric_exp_hist_snapshot_destroy(&snapshot); +} + +static struct cmt *decode_all(char *buffer, size_t size, size_t *offset) +{ + struct cmt *decoded = NULL; + int result; + + result = cmt_decode_msgpack_create(&decoded, buffer, size, offset); + TEST_CHECK_(result == CMT_DECODE_MSGPACK_SUCCESS, "decode result %d", result); + return result == CMT_DECODE_MSGPACK_SUCCESS ? decoded : NULL; +} + +/* Chunks written by released agents decode completely in this release, keep + * their content through a re-encode (forward relay) and through cmt_cat. */ +static void test_release_fixtures(void) +{ + size_t index; + size_t offset; + size_t size; + cfl_sds_t fixture; + struct cmt *decoded; + struct cmt *again; + struct cmt *merged; + char *buffer; + char label[128]; + + for (index = 0; index < sizeof(release_fixtures) / sizeof(*release_fixtures); index++) { + fixture = read_fixture(release_fixtures[index]); + TEST_ASSERT(fixture != NULL); + + offset = 0; + decoded = decode_all(fixture, cfl_sds_len(fixture), &offset); + TEST_ASSERT(decoded != NULL); + TEST_CHECK(offset == cfl_sds_len(fixture)); + snprintf(label, sizeof(label), "%s decoded", release_fixtures[index]); + check_release_content(decoded, RELEASE_FIXTURE_HAS_UNIT(index), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index), label); + + TEST_ASSERT(cmt_encode_msgpack_create(decoded, &buffer, &size) == 0); + offset = 0; + again = decode_all(buffer, size, &offset); + TEST_ASSERT(again != NULL); + snprintf(label, sizeof(label), "%s re-encoded", release_fixtures[index]); + check_release_content(again, RELEASE_FIXTURE_HAS_UNIT(index), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index), label); + + merged = cmt_create(); + TEST_ASSERT(merged != NULL); + TEST_CHECK(cmt_cat(merged, again) == 0); + snprintf(label, sizeof(label), "%s concatenated", release_fixtures[index]); + check_release_content(merged, RELEASE_FIXTURE_HAS_UNIT(index), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(index), label); + + cmt_destroy(merged); + cmt_decode_msgpack_destroy(again); + cmt_encode_msgpack_destroy(buffer); + cmt_decode_msgpack_destroy(decoded); + cfl_sds_destroy(fixture); + } +} + +/* A forward chunk can hold contexts from agents of different releases. */ +static void test_mixed_release_batch(void) +{ + size_t index; + size_t offset; + size_t count; + size_t size; + cfl_sds_t batch; + cfl_sds_t fixture; + struct cmt *decoded; + char *buffer; + + batch = cfl_sds_create_size(0); + TEST_ASSERT(batch != NULL); + for (index = 0; index < sizeof(release_fixtures) / sizeof(*release_fixtures); index++) { + fixture = read_fixture(release_fixtures[index]); + TEST_ASSERT(fixture != NULL); + batch = cfl_sds_cat(batch, fixture, cfl_sds_len(fixture)); + cfl_sds_destroy(fixture); + } + + /* The current encoder's output for the newest fixture closes the batch. */ + fixture = read_fixture(release_fixtures[2]); + offset = 0; + decoded = decode_all(fixture, cfl_sds_len(fixture), &offset); + TEST_ASSERT(decoded != NULL); + TEST_ASSERT(cmt_encode_msgpack_create(decoded, &buffer, &size) == 0); + batch = cfl_sds_cat(batch, buffer, size); + cmt_encode_msgpack_destroy(buffer); + cmt_decode_msgpack_destroy(decoded); + cfl_sds_destroy(fixture); + + offset = 0; + count = 0; + while (offset < cfl_sds_len(batch)) { + decoded = decode_all(batch, cfl_sds_len(batch), &offset); + TEST_ASSERT(decoded != NULL); + check_release_content(decoded, RELEASE_FIXTURE_HAS_UNIT(count), + RELEASE_FIXTURE_HAS_HISTOGRAM_TEMPORALITY(count), "batch entry"); + cmt_decode_msgpack_destroy(decoded); + count++; + } + TEST_CHECK(count == 4); + TEST_CHECK(offset == cfl_sds_len(batch)); + cfl_sds_destroy(batch); +} + +/* Every metric type with labels, static samples, typed values, start + * timestamps, units, OTLP metadata and the native metadata sidecar. */ +static struct cmt *create_full_context(void) +{ + struct cmt *ctx; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_histogram *histogram; + struct cmt_summary *summary; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *sample; + struct cfl_kvlist *metadata; + double quantiles[] = {0.5, 0.99}; + double quantile_values[] = {1, 2}; + uint64_t buckets[] = {1, 2, 3}; + uint64_t positive[] = {4, 5}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + TEST_ASSERT(cmt_label_add(ctx, "dc", "eu-1") == 0); + + counter = cmt_counter_create(ctx, "ns", "sub", "events_total", "events", 2, + (char *[]) {"a", "b"}); + TEST_ASSERT(counter != NULL); + cmt_counter_allow_reset(counter); + counter->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + TEST_ASSERT(cmt_counter_set(counter, 10, 5, 2, (char *[]) {"x", "y"}) == 0); + sample = cmt_map_metric_get(&counter->opts, counter->map, 2, (char *[]) {"x", "y"}, CMT_FALSE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_start_timestamp(sample, 5); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", 1) == 0); + counter->map->unit = cfl_sds_create("{event}"); + + gauge = cmt_gauge_create(ctx, "", "", "level", "level", 0, NULL); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 10, 0, 0, NULL) == 0); + cmt_metric_set_int64(&gauge->map->metric, 10, INT64_MIN); + gauge->map->otlp_metadata = cfl_kvlist_create(); + metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(metadata, "k", "v") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", metadata) == 0); + + untyped = cmt_untyped_create(ctx, "", "", "raw", "raw", 1, (char *[]) {"k"}); + TEST_ASSERT(untyped != NULL); + TEST_ASSERT(cmt_untyped_set(untyped, 10, NAN, 1, (char *[]) {"nan"}) == 0); + + histogram = cmt_histogram_create(ctx, "", "", "latency", "latency", + cmt_histogram_buckets_create(2, 1.0, 2.0), + 1, (char *[]) {"r"}); + TEST_ASSERT(histogram != NULL); + TEST_ASSERT(cmt_histogram_set_default(histogram, 10, buckets, 4.5, 3, 1, + (char *[]) {"/"}) == 0); + + summary = cmt_summary_create(ctx, "", "", "rpc", "rpc", 2, quantiles, 0, NULL); + TEST_ASSERT(summary != NULL); + TEST_ASSERT(cmt_summary_set_default(summary, 10, quantile_values, 3, 2, 0, NULL) == 0); + + exp_histogram = cmt_exp_histogram_create(ctx, "", "", "exp", "exp", 0, NULL); + TEST_ASSERT(exp_histogram != NULL); + TEST_ASSERT(cmt_exp_histogram_set_default(exp_histogram, 10, -2, 0, 0, 3, 2, positive, + 0, 0, NULL, 0, 0, 9, 0, NULL) == 0); + return ctx; +} + +/* Released decoders (Fluent Bit 5.0.11, 5.1.x) accept exactly these keys; + * Fluent Bit 5.0.0 accepts all of them except "unit". */ +static const char *released_keys[] = { + "aggregation_type", "buckets", "cmetrics", "count", "desc", "exp_histogram", + "external", "hash", "histogram", "labels", "meta", "metrics", "name", + "negative_buckets", "negative_offset", "ns", "opts", "positive_buckets", + "positive_offset", "processing", "quantiles", "quantiles_set", "scale", "ss", + "start_ts", "static_labels", "sum", "summary", "sum_set", "ts", "type", "unit", + "value", "values", "value_type", "value_int64", "value_uint64", "ver", + "zero_count", "zero_threshold", NULL +}; + +static int is_released_key(const char *key, size_t length) +{ + size_t index; + + for (index = 0; released_keys[index] != NULL; index++) { + if (strlen(released_keys[index]) == length && + memcmp(released_keys[index], key, length) == 0) { + return CMT_TRUE; + } + } + return CMT_FALSE; +} + +/* Walk a value; map keys are schema keys except inside the schemaless + * internal ("cmetrics") and external metadata maps, which are skipped. */ +static int walk_keys(mpack_reader_t *reader, int depth) +{ + mpack_tag_t tag; + uint32_t index; + uint32_t count; + char key[64]; + uint32_t length; + int unknown = 0; + + tag = mpack_read_tag(reader); + if (mpack_reader_error(reader) != mpack_ok) { + return -1; + } + switch (mpack_tag_type(&tag)) { + case mpack_type_map: + count = mpack_tag_map_count(&tag); + for (index = 0; index < count; index++) { + tag = mpack_read_tag(reader); + if (mpack_tag_type(&tag) != mpack_type_str) { + return -1; + } + length = mpack_tag_str_length(&tag); + if (length >= sizeof(key)) { + return -1; + } + mpack_read_bytes(reader, key, length); + mpack_done_str(reader); + if (!is_released_key(key, length)) { + TEST_MSG("key not known to released readers: %.*s", (int) length, key); + unknown++; + } + if ((length == 8 && memcmp(key, "cmetrics", 8) == 0) || + (length == 8 && memcmp(key, "external", 8) == 0)) { + mpack_discard(reader); + } + else { + unknown += walk_keys(reader, depth + 1); + } + } + mpack_done_map(reader); + break; + case mpack_type_array: + count = mpack_tag_array_count(&tag); + for (index = 0; index < count; index++) { + unknown += walk_keys(reader, depth + 1); + } + mpack_done_array(reader); + break; + case mpack_type_str: + mpack_skip_bytes(reader, mpack_tag_str_length(&tag)); + mpack_done_str(reader); + break; + case mpack_type_bin: + mpack_skip_bytes(reader, mpack_tag_bin_length(&tag)); + mpack_done_bin(reader); + break; + default: + break; + } + return unknown; +} + +/* New information must travel in the schemaless metadata maps: a new schema + * key makes every released reader reject the whole chunk. */ +static void test_encoder_uses_released_keys_only(void) +{ + struct cmt *ctx; + mpack_reader_t reader; + char *buffer; + size_t size; + int unknown; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + mpack_reader_init_data(&reader, buffer, size); + unknown = walk_keys(&reader, 0); + TEST_CHECK(mpack_reader_destroy(&reader) == mpack_ok); + TEST_CHECK_(unknown == 0, "%d keys unknown to released readers", unknown); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static uint64_t double_bits(double value) +{ + uint64_t bits; + + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +/* Exact values of every scalar type for counters, gauges and untyped. */ +static void test_scalar_extremes(void) +{ + static const uint64_t double_patterns[] = { + 0x7ff8000000000000ULL, /* quiet NaN */ + 0x7ff0000000000002ULL, /* Prometheus staleness marker */ + 0x7ff0000000000000ULL, /* +Inf */ + 0xfff0000000000000ULL, /* -Inf */ + 0x8000000000000000ULL, /* -0.0 */ + 0x0000000000000001ULL, /* smallest denormal */ + 0x7fefffffffffffffULL, /* DBL_MAX */ + }; + static const int64_t signed_values[] = {INT64_MIN, INT64_MAX, -1, 0}; + static const uint64_t unsigned_values[] = {UINT64_MAX, (uint64_t) INT64_MAX + 1, 0}; + struct cmt *ctx; + struct cmt *decoded; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_untyped *untyped; + struct cmt_metric *sample; + struct cmt_map *maps[3]; + char value_label[32]; + char *labels[] = {value_label}; + char *buffer; + size_t size; + size_t offset; + size_t index; + size_t type; + double value; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + counter = cmt_counter_create(ctx, "", "", "c", "c", 1, (char *[]) {"v"}); + gauge = cmt_gauge_create(ctx, "", "", "g", "g", 1, (char *[]) {"v"}); + untyped = cmt_untyped_create(ctx, "", "", "u", "u", 1, (char *[]) {"v"}); + TEST_ASSERT(counter != NULL && gauge != NULL && untyped != NULL); + maps[0] = counter->map; + maps[1] = gauge->map; + maps[2] = untyped->map; + + for (type = 0; type < 3; type++) { + for (index = 0; index < sizeof(double_patterns) / sizeof(*double_patterns); index++) { + snprintf(value_label, sizeof(value_label), "d%zu", index); + sample = cmt_map_metric_get(&counter->opts, maps[type], 1, labels, CMT_TRUE); + TEST_ASSERT(sample != NULL); + memcpy(&value, &double_patterns[index], sizeof(value)); + cmt_metric_set_double(sample, 1, value); + } + for (index = 0; index < sizeof(signed_values) / sizeof(*signed_values); index++) { + snprintf(value_label, sizeof(value_label), "i%zu", index); + sample = cmt_map_metric_get(&counter->opts, maps[type], 1, labels, CMT_TRUE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_int64(sample, 1, signed_values[index]); + } + for (index = 0; index < sizeof(unsigned_values) / sizeof(*unsigned_values); index++) { + snprintf(value_label, sizeof(value_label), "u%zu", index); + sample = cmt_map_metric_get(&counter->opts, maps[type], 1, labels, CMT_TRUE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_uint64(sample, 1, unsigned_values[index]); + } + } + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + offset = 0; + decoded = decode_all(buffer, size, &offset); + TEST_ASSERT(decoded != NULL); + maps[0] = find_map(decoded, CMT_COUNTER, "c"); + maps[1] = find_map(decoded, CMT_GAUGE, "g"); + maps[2] = find_map(decoded, CMT_UNTYPED, "u"); + for (type = 0; type < 3; type++) { + TEST_ASSERT(maps[type] != NULL); + for (index = 0; index < sizeof(double_patterns) / sizeof(*double_patterns); index++) { + snprintf(value_label, sizeof(value_label), "d%zu", index); + sample = find_sample(maps[type], value_label); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_DOUBLE); + TEST_CHECK_(double_bits(cmt_metric_get_value(sample)) == double_patterns[index], + "type %zu double %016" PRIx64, type, double_patterns[index]); + } + for (index = 0; index < sizeof(signed_values) / sizeof(*signed_values); index++) { + snprintf(value_label, sizeof(value_label), "i%zu", index); + sample = find_sample(maps[type], value_label); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_INT64); + TEST_CHECK_(cmt_metric_get_int64_value(sample) == signed_values[index], + "type %zu int64 %" PRId64, type, signed_values[index]); + } + for (index = 0; index < sizeof(unsigned_values) / sizeof(*unsigned_values); index++) { + snprintf(value_label, sizeof(value_label), "u%zu", index); + sample = find_sample(maps[type], value_label); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_UINT64); + TEST_CHECK_(cmt_metric_get_uint64_value(sample) == unsigned_values[index], + "type %zu uint64 %" PRIu64, type, unsigned_values[index]); + } + } + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Storage corruption or a broken peer: every prefix and every single byte + * corruption of a full chunk is rejected or decoded, never crashes, and a + * decoded context can be encoded again. Run under Valgrind or ASan. */ +static void test_corrupted_chunks(void) +{ + struct cmt *ctx; + struct cmt *decoded; + char *buffer; + char *copy; + char *again; + size_t size; + size_t again_size; + size_t length; + size_t index; + size_t offset; + int result; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + copy = malloc(size); + TEST_ASSERT(copy != NULL); + + for (length = 0; length < size; length++) { + offset = 0; + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, buffer, length, &offset); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + cmt_decode_msgpack_destroy(decoded); + } + } + + for (index = 0; index < size; index++) { + memcpy(copy, buffer, size); + copy[index] ^= 0xa5; + offset = 0; + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, copy, size, &offset); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + TEST_CHECK(offset <= size); + if (cmt_encode_msgpack_create(decoded, &again, &again_size) == 0) { + cmt_encode_msgpack_destroy(again); + } + cmt_decode_msgpack_destroy(decoded); + } + } + + free(copy); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Replace the array header that follows the nth occurrence of a key. */ +static char *replace_array_header(char *buffer, size_t size, const char *key, + const char *header, size_t header_size, size_t *out_size) +{ + size_t index; + size_t key_length; + char *patched; + char encoded_key[32]; + + key_length = strlen(key); + encoded_key[0] = (char) (0xa0 | key_length); + memcpy(&encoded_key[1], key, key_length); + for (index = 0; index + key_length + 2 <= size; index++) { + if (memcmp(&buffer[index], encoded_key, key_length + 1) != 0) { + continue; + } + index += key_length + 1; + /* fixarray headers only: the fixture arrays are small */ + if (((unsigned char) buffer[index] & 0xf0) != 0x90) { + return NULL; + } + patched = malloc(size + header_size); + TEST_ASSERT(patched != NULL); + memcpy(patched, buffer, index); + memcpy(&patched[index], header, header_size); + memcpy(&patched[index + header_size], &buffer[index + 1], size - index - 1); + *out_size = size + header_size - 1; + return patched; + } + return NULL; +} + +/* A declared element count far beyond the remaining input must be rejected + * before any allocation sized from it. */ +static void test_oversized_declared_counts(void) +{ + static const char *keys[] = {"values", "labels", "buckets", "quantiles"}; + static const char huge_array[] = {(char) 0xdd, (char) 0xff, (char) 0xff, + (char) 0xff, (char) 0xff}; + struct cmt *ctx; + struct cmt *decoded; + char *buffer; + char *patched; + size_t size; + size_t patched_size; + size_t offset; + size_t index; + int result; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + for (index = 0; index < sizeof(keys) / sizeof(*keys); index++) { + patched = replace_array_header(buffer, size, keys[index], huge_array, + sizeof(huge_array), &patched_size); + TEST_ASSERT_(patched != NULL, "array key %s found", keys[index]); + offset = 0; + decoded = NULL; + result = cmt_decode_msgpack_create(&decoded, patched, patched_size, &offset); + TEST_CHECK_(result != CMT_DECODE_MSGPACK_SUCCESS, "%s count rejected", keys[index]); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + cmt_decode_msgpack_destroy(decoded); + } + free(patched); + } + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Released readers reject any other format version, so the version cannot + * be raised to signal new content; this release must behave the same. */ +static void test_format_version_is_strict(void) +{ + static const char version_key[] = {(char) 0xa3, 'v', 'e', 'r', 0x02}; + struct cmt *ctx; + struct cmt *decoded; + char *buffer; + size_t size; + size_t index; + size_t offset; + int found = CMT_FALSE; + + ctx = create_full_context(); + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + for (index = 0; index + sizeof(version_key) <= size; index++) { + if (memcmp(&buffer[index], version_key, sizeof(version_key)) == 0) { + buffer[index + sizeof(version_key) - 1] = 0x03; + found = CMT_TRUE; + break; + } + } + TEST_ASSERT(found); + offset = 0; + decoded = NULL; + TEST_CHECK(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) != + CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(decoded == NULL); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +TEST_LIST = { + {"release_fixtures", test_release_fixtures}, + {"mixed_release_batch", test_mixed_release_batch}, + {"encoder_uses_released_keys_only", test_encoder_uses_released_keys_only}, + {"scalar_extremes", test_scalar_extremes}, + {"corrupted_chunks", test_corrupted_chunks}, + {"oversized_declared_counts", test_oversized_declared_counts}, + {"format_version_is_strict", test_format_version_is_strict}, + {0} +}; diff --git a/tests/opentelemetry.c b/tests/opentelemetry.c index 0bf8221e..074b8f52 100644 --- a/tests/opentelemetry.c +++ b/tests/opentelemetry.c @@ -3220,7 +3220,804 @@ static void test_opentelemetry_repeated_unlabeled_points_last_wins(void) cfl_sds_destroy(payload); } +/* Empty nested AnyValues are optional metadata, not allocation failures. */ +static void test_otlp_nested_empty_attributes_keep_point(void) +{ + int mode; + size_t offset; + size_t size; + int result; + cfl_sds_t payload; + cfl_sds_t wire; + struct cfl_list contexts; + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Common__V1__AnyValue empty; + Opentelemetry__Proto__Common__V1__AnyValue nested; + Opentelemetry__Proto__Common__V1__AnyValue *items[] = {&empty}; + Opentelemetry__Proto__Common__V1__ArrayValue array; + Opentelemetry__Proto__Common__V1__KeyValue entry; + Opentelemetry__Proto__Common__V1__KeyValue *entries[] = {&entry}; + Opentelemetry__Proto__Common__V1__KeyValueList kvlist; + Opentelemetry__Proto__Common__V1__KeyValue *attribute; + Opentelemetry__Proto__Common__V1__AnyValue *original; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + /* An array containing an empty value, a kvlist with no value, and a + * kvlist containing an explicitly empty value must all remain usable. */ + for (mode = 0; mode < 3; mode++) { + payload = generate_gauge_int_otlp_payload_with_many_attributes(2); + TEST_ASSERT(payload != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(payload), (const uint8_t *) payload); + TEST_ASSERT(request != NULL); + cfl_sds_destroy(payload); + attribute = request->resource_metrics[0]->scope_metrics[0]->metrics[0]->gauge->data_points[0]->attributes[0]; + original = attribute->value; + opentelemetry__proto__common__v1__any_value__init(&empty); + opentelemetry__proto__common__v1__any_value__init(&nested); + opentelemetry__proto__common__v1__array_value__init(&array); + opentelemetry__proto__common__v1__key_value__init(&entry); + opentelemetry__proto__common__v1__key_value_list__init(&kvlist); + array.n_values = 1; + array.values = items; + entry.key = "empty"; + entry.value = mode == 2 ? &empty : NULL; + kvlist.n_values = 1; + kvlist.values = entries; + if (mode == 0) { + nested.value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE; + nested.array_value = &array; + } + else { + nested.value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE; + nested.kvlist_value = &kvlist; + } + attribute->value = &nested; + size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(request); + payload = cfl_sds_create_size(size); + TEST_ASSERT(payload != NULL); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(request, (uint8_t *) payload); + cfl_sds_len_set(payload, size); + attribute->value = original; + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + + offset = 0; + result = cmt_decode_opentelemetry_create(&contexts, payload, size, &offset); + TEST_CHECK(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + ctx = cfl_list_entry_first(&contexts, struct cmt, _head); + TEST_ASSERT(ctx != NULL); + gauge = cfl_list_entry_first(&ctx->gauges, struct cmt_gauge, _head); + TEST_ASSERT(gauge != NULL); + sample = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_ASSERT(sample != NULL); + TEST_CHECK(cmt_metric_get_int64_value(sample) == 1); + TEST_CHECK(cmt_metric_get_timestamp(sample) == 123); + TEST_CHECK(cfl_kvlist_fetch(sample->otlp_metadata, "attributes") == NULL); + wire = cmt_encode_opentelemetry_create(ctx); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "g_many_attrs"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->as_int == 1); + TEST_ASSERT(metric->gauge->data_points[0]->n_attributes == 1); + TEST_CHECK(strcmp(metric->gauge->data_points[0]->attributes[0]->key, "attr_001") == 0); + TEST_CHECK(metric->gauge->data_points[0]->attributes[0]->value->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE); + TEST_CHECK(strcmp(metric->gauge->data_points[0]->attributes[0]->value->string_value, + "value_001") == 0); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_decode_opentelemetry_destroy(&contexts); + } + cfl_sds_destroy(payload); + } +} + +static void test_otlp_restore_preserves_legacy_only_family(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *legacy; + struct cmt_gauge *native; + struct cfl_kvlist *external = NULL; + struct cfl_variant *node; + mpack_writer_t writer; + mpack_reader_t reader; + char *buffer = NULL; + size_t size = 0; + size_t offset = 0; + cfl_sds_t wire; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + legacy = cmt_gauge_create(ctx, "", "", "legacy_only", "legacy", 0, NULL); + native = cmt_gauge_create(ctx, "", "", "native", "native", 0, NULL); + TEST_ASSERT(legacy != NULL && native != NULL); + TEST_ASSERT(cmt_gauge_set(legacy, 100, 7, 0, NULL) == 0); + TEST_ASSERT(cmt_gauge_set(native, 200, 9, 0, NULL) == 0); + legacy->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(legacy->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(legacy->map->metric.otlp_metadata, "flags", 1) == 0); + + /* Build an external legacy tree, then leave this family with no native + * metadata. The other family forces a native sidecar for both maps. */ + mpack_writer_init_growable(&writer, &buffer, &size); + TEST_ASSERT(cmt_otlp_metadata_pack_external(&writer, ctx) == 0); + TEST_ASSERT(mpack_writer_destroy(&writer) == mpack_ok); + mpack_reader_init_data(&reader, buffer, size); + TEST_ASSERT(unpack_cfl_kvlist(&reader, &external) == 0); + TEST_ASSERT(mpack_reader_destroy(&reader) == mpack_ok); + free(buffer); + cfl_kvlist_destroy(ctx->external_metadata); + ctx->external_metadata = external; + cfl_kvlist_destroy(legacy->map->metric.otlp_metadata); + legacy->map->metric.otlp_metadata = NULL; + native->map->metric.otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(native->map->metric.otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(native->map->metric.otlp_metadata, "flags", 2) == 0); + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, &buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, buffer, size, &offset) == 0); + node = cfl_kvlist_fetch(decoded->external_metadata, "otlp"); + TEST_ASSERT(node != NULL); + node = cfl_kvlist_fetch(node->data.as_kvlist, "metrics"); + TEST_ASSERT(node != NULL); + node = cfl_kvlist_fetch(node->data.as_kvlist, "gauge"); + TEST_ASSERT(node != NULL); + TEST_CHECK(cfl_kvlist_fetch(node->data.as_kvlist, "legacy_only") != NULL); + TEST_CHECK(cfl_kvlist_fetch(node->data.as_kvlist, "native") == NULL); + + wire = cmt_encode_opentelemetry_create(decoded); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "legacy_only"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->flags == 1); + metric = find_otlp_metric_by_name(request, "native"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 1); + TEST_CHECK(metric->gauge->data_points[0]->flags == 2); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* + * Released readers decode the sidecar and the legacy tree as opaque context + * metadata and write them back unchanged, even after editing points. These + * helpers capture both as such a relay would see them. + */ +static struct cfl_kvlist *relay_snapshot(struct cmt *ctx, int internal) +{ + struct cfl_kvlist *copy = NULL; + mpack_writer_t writer; + mpack_reader_t reader; + char *buffer = NULL; + size_t size = 0; + int result; + + mpack_writer_init_growable(&writer, &buffer, &size); + if (internal) { + result = cmt_otlp_metadata_pack(&writer, ctx); + } + else { + result = cmt_otlp_metadata_pack_external(&writer, ctx); + } + if (mpack_writer_destroy(&writer) != mpack_ok || result != 0) { + free(buffer); + return NULL; + } + mpack_reader_init_data(&reader, buffer, size); + result = unpack_cfl_kvlist(&reader, ©); + if (mpack_reader_destroy(&reader) != mpack_ok || result != 0) { + cfl_kvlist_destroy(copy); + copy = NULL; + } + free(buffer); + return copy; +} + +/* Replace native metadata with the captured opaque copies, as a released + * relay holds nothing else. */ +static void relay_install(struct cmt *ctx, struct cmt_map *map, + struct cfl_kvlist *internal, struct cfl_kvlist *external) +{ + struct cfl_list *head; + struct cmt_metric *sample; + + cfl_kvlist_destroy(map->otlp_metadata); + map->otlp_metadata = NULL; + cfl_kvlist_destroy(map->metric.otlp_metadata); + map->metric.otlp_metadata = NULL; + cfl_list_foreach(head, &map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = NULL; + } + cfl_kvlist_destroy(ctx->internal_metadata); + ctx->internal_metadata = internal; + cfl_kvlist_destroy(ctx->external_metadata); + ctx->external_metadata = external; +} + +/* Remove the point identifiers, producing a v2.3.0 sidecar. */ +static void relay_strip_point_ids(struct cfl_kvlist *internal) +{ + struct cfl_variant *families; + size_t index; + + families = cfl_kvlist_fetch(internal, "__cmetrics_native_metadata_v1"); + TEST_ASSERT(families != NULL && families->type == CFL_VARIANT_ARRAY); + for (index = 0; index < families->data.as_array->entry_count; index++) { + cfl_kvlist_remove(families->data.as_array->entries[index]->data.as_kvlist, + "point_ids"); + } +} + +static struct cmt_gauge *relay_gauge(struct cmt *ctx, uint64_t flags_a, uint64_t flags_b, + int family_metadata) +{ + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *metadata; + char *values[2][1] = {{"a"}, {"b"}}; + uint64_t flags[2]; + int index; + + flags[0] = flags_a; + flags[1] = flags_b; + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 1, (char *[]) {"site"}); + TEST_ASSERT(gauge != NULL); + for (index = 0; index < 2; index++) { + TEST_ASSERT(cmt_gauge_set(gauge, 100, index + 1, 1, values[index]) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, values[index], CMT_FALSE); + TEST_ASSERT(sample != NULL); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", flags[index]) == 0); + } + if (family_metadata) { + gauge->map->otlp_metadata = cfl_kvlist_create(); + metadata = cfl_kvlist_create(); + TEST_ASSERT(gauge->map->otlp_metadata != NULL && metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(metadata, "owner", "relay") == 0); + TEST_ASSERT(cfl_kvlist_insert_kvlist(gauge->map->otlp_metadata, "metadata", metadata) == 0); + } + return gauge; +} + +static struct cmt *relay_round_trip(struct cmt *ctx, char **buffer) +{ + struct cmt *decoded = NULL; + size_t size; + size_t offset = 0; + + TEST_ASSERT(cmt_encode_msgpack_create(ctx, buffer, &size) == 0); + TEST_ASSERT(cmt_decode_msgpack_create(&decoded, *buffer, size, &offset) == 0); + TEST_CHECK(offset == size); + return decoded; +} + +static int relay_otlp_flags(struct cmt *ctx, char *site, uint32_t *flags) +{ + cfl_sds_t wire; + size_t index; + int found = CMT_FALSE; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *point; + + wire = cmt_encode_opentelemetry_create(ctx); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "relay"); + TEST_ASSERT(metric != NULL && metric->gauge != NULL); + for (index = 0; index < metric->gauge->n_data_points; index++) { + point = metric->gauge->data_points[index]; + if (point->n_attributes == 1 && + strcmp(point->attributes[0]->value->string_value, site) == 0) { + *flags = point->flags; + found = CMT_TRUE; + } + } + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + return found; +} + +/* A released relay replaces series "a" with "c" without updating the sidecar. + * Metadata recorded for "a" must not be attached to another series. */ +static void test_otlp_restore_binds_points_by_identity(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 1, 0, CMT_FALSE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + cmt_map_metric_destroy(cmt_map_metric_get(&gauge->opts, gauge->map, 1, + (char *[]) {"a"}, CMT_FALSE)); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 3, 1, (char *[]) {"c"}) == 0); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 0); + TEST_CHECK(relay_otlp_flags(decoded, "c", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* A released relay records a new observation for "b" without updating the + * sidecar. Metadata of the previous observation must not be restored. */ +static void test_otlp_restore_drops_stale_observation(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 0, 1, CMT_FALSE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + TEST_ASSERT(cmt_gauge_set(gauge, 200, 9, 1, (char *[]) {"b"}) == 0); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 0); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* A relay that reverses a large family must not make restoration scan the + * identifiers once per point, and every point keeps its own metadata. */ +static void test_otlp_restore_binds_reordered_points(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + struct cfl_variant *flags; + struct cfl_list reversed; + struct cfl_list *head; + struct cfl_list *tmp; + struct cmt_map_label *label; + char value[16]; + char *values[1] = {value}; + char *buffer; + int index; + int bound = 0; + const int count = 2000; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 1, (char *[]) {"site"}); + TEST_ASSERT(gauge != NULL); + for (index = 0; index < count; index++) { + snprintf(value, sizeof(value), "%d", index); + TEST_ASSERT(cmt_gauge_set(gauge, 100, index, 1, values) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, values, CMT_FALSE); + TEST_ASSERT(sample != NULL); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", index) == 0); + } + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_install(ctx, gauge->map, internal, external); + + cfl_list_init(&reversed); + cfl_list_foreach_safe(head, tmp, &gauge->map->metrics) { + cfl_list_del(head); + cfl_list_prepend(head, &reversed); + } + cfl_list_foreach_safe(head, tmp, &reversed) { + cfl_list_del(head); + cfl_list_add(head, &gauge->map->metrics); + } + + decoded = relay_round_trip(ctx, &buffer); + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + cfl_list_foreach(head, &gauge->map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + label = cfl_list_entry_first(&sample->labels, struct cmt_map_label, _head); + flags = cfl_kvlist_fetch(sample->otlp_metadata, "flags"); + if (flags != NULL && + (uint64_t) (flags->type == CFL_VARIANT_UINT ? flags->data.as_uint64 : + (uint64_t) flags->data.as_int64) == strtoull(label->name, NULL, 10)) { + bound++; + } + } + TEST_CHECK(bound == count); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* The series hash concatenates label values without boundaries, so these + * two series share it. A released relay drops the first: its metadata must + * not move to the survivor. */ +static void test_otlp_restore_binds_points_by_label_values(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_metric *first; + struct cmt_metric *second; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + struct cfl_variant *flags; + char *buffer; + char *a[] = {"ab", "c"}; + char *b[] = {"a", "bc"}; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 2, (char *[]) {"x", "y"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 1, 2, a) == 0); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 2, 2, b) == 0); + first = cmt_map_metric_get(&gauge->opts, gauge->map, 2, a, CMT_FALSE); + second = cmt_map_metric_get(&gauge->opts, gauge->map, 2, b, CMT_FALSE); + TEST_ASSERT(first != NULL && second != NULL && first != second); + TEST_ASSERT(first->hash == second->hash); + first->otlp_metadata = cfl_kvlist_create(); + second->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(first->otlp_metadata != NULL && second->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(first->otlp_metadata, "flags", 1) == 0); + TEST_ASSERT(cfl_kvlist_insert_uint64(second->otlp_metadata, "flags", 0) == 0); + + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_install(ctx, gauge->map, internal, external); + cmt_map_metric_destroy(first); + + decoded = relay_round_trip(ctx, &buffer); + gauge = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_ASSERT(cfl_list_size(&gauge->map->metrics) == 1); + second = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + flags = cfl_kvlist_fetch(second->otlp_metadata, "flags"); + TEST_ASSERT(flags != NULL); + TEST_CHECK((flags->type == CFL_VARIANT_UINT && flags->data.as_uint64 == 0) || + (flags->type == CFL_VARIANT_INT && flags->data.as_int64 == 0)); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* One window of series "a" at time 100, as an OTLP point with its own + * start time. */ +static struct cmt *relay_window(uint64_t start, uint64_t flags, double value) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 1, (char *[]) {"site"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, value, 1, (char *[]) {"a"}) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, (char *[]) {"a"}, CMT_FALSE); + TEST_ASSERT(sample != NULL); + cmt_metric_set_start_timestamp(sample, start); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", flags) == 0); + return ctx; +} + +/* Series "a" with point metadata nested beyond the decoder's depth limit, + * relayed by a released reader, optionally after changing its value at the + * same time. */ +static struct cmt *relay_unreadable(int change_value, char **buffer) +{ + struct cmt *ctx; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *inner; + struct cfl_kvlist *outer; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + struct cmt *decoded; + int depth; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 1, 0, CMT_FALSE); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 1, (char *[]) {"a"}, CMT_FALSE); + TEST_ASSERT(sample != NULL); + inner = cfl_kvlist_create(); + TEST_ASSERT(inner != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(inner, "leaf", "v") == 0); + for (depth = 0; depth < 40; depth++) { + outer = cfl_kvlist_create(); + TEST_ASSERT(outer != NULL); + TEST_ASSERT(cfl_kvlist_insert_kvlist(outer, "nested", inner) == 0); + inner = outer; + } + TEST_ASSERT(cfl_kvlist_insert_kvlist(sample->otlp_metadata, "nested", inner) == 0); + + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_install(ctx, gauge->map, internal, external); + if (change_value) { + TEST_ASSERT(cmt_gauge_set(gauge, 100, 99, 1, (char *[]) {"a"}) == 0); + } + decoded = relay_round_trip(ctx, buffer); + cmt_destroy(ctx); + return decoded; +} + +enum shared_key_sidecar { + SHARED_KEY_CURRENT, /* restored */ + SHARED_KEY_V230, /* no identifiers, point count changed: skipped */ + SHARED_KEY_NEW_FAMILY /* relay added a family: rejected */ +}; + +/* A released relay adds a series whose labels share the series hash of a + * recorded point, at the same time. That point's legacy entry, its only + * readable metadata, can belong to either: neither may take it, whether the + * native sidecar is restored, skipped or rejected. */ +static void check_shared_legacy_key(enum shared_key_sidecar sidecar) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *added; + struct cmt_metric *sample; + struct cfl_kvlist *inner; + struct cfl_kvlist *outer; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + cfl_sds_t wire; + char *buffer; + char *a[] = {"ab", "c"}; + char *b[] = {"a", "bc"}; + size_t index; + int depth; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *request; + Opentelemetry__Proto__Metrics__V1__Metric *metric; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = cmt_gauge_create(ctx, "", "", "relay", "relay", 2, (char *[]) {"x", "y"}); + TEST_ASSERT(gauge != NULL); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 1, 2, a) == 0); + sample = cmt_map_metric_get(&gauge->opts, gauge->map, 2, a, CMT_FALSE); + TEST_ASSERT(sample != NULL); + sample->otlp_metadata = cfl_kvlist_create(); + TEST_ASSERT(sample->otlp_metadata != NULL); + TEST_ASSERT(cfl_kvlist_insert_uint64(sample->otlp_metadata, "flags", 1) == 0); + inner = cfl_kvlist_create(); + TEST_ASSERT(inner != NULL); + TEST_ASSERT(cfl_kvlist_insert_string(inner, "leaf", "v") == 0); + for (depth = 0; depth < 40; depth++) { + outer = cfl_kvlist_create(); + TEST_ASSERT(outer != NULL); + TEST_ASSERT(cfl_kvlist_insert_kvlist(outer, "nested", inner) == 0); + inner = outer; + } + TEST_ASSERT(cfl_kvlist_insert_kvlist(sample->otlp_metadata, "nested", inner) == 0); + + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + if (sidecar == SHARED_KEY_V230) { + relay_strip_point_ids(internal); + } + relay_install(ctx, gauge->map, internal, external); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 2, 2, b) == 0); + TEST_ASSERT(cmt_map_metric_get(&gauge->opts, gauge->map, 2, b, CMT_FALSE)->hash == + sample->hash); + if (sidecar == SHARED_KEY_NEW_FAMILY) { + added = cmt_gauge_create(ctx, "", "", "added", "added", 0, NULL); + TEST_ASSERT(added != NULL); + TEST_ASSERT(cmt_gauge_set(added, 100, 3, 0, NULL) == 0); + } + + decoded = relay_round_trip(ctx, &buffer); + wire = cmt_encode_opentelemetry_create(decoded); + TEST_ASSERT(wire != NULL); + request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_ASSERT(request != NULL); + metric = find_otlp_metric_by_name(request, "relay"); + TEST_ASSERT(metric != NULL && metric->gauge->n_data_points == 2); + for (index = 0; index < metric->gauge->n_data_points; index++) { + TEST_CHECK_(metric->gauge->data_points[index]->flags == 0, + "sidecar %d: point %g takes no shared legacy flags", sidecar, + metric->gauge->data_points[index]->as_double); + } + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(request, NULL); + cmt_encode_opentelemetry_destroy(wire); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +static void test_otlp_restore_drops_shared_legacy_key(void) +{ + check_shared_legacy_key(SHARED_KEY_CURRENT); + check_shared_legacy_key(SHARED_KEY_V230); + check_shared_legacy_key(SHARED_KEY_NEW_FAMILY); +} + +/* Native point metadata that cannot be read back leaves the point its + * legacy entry, but only while its identity still matches: a changed value + * must not take the flags recorded for the old one. */ +static void test_otlp_restore_keeps_legacy_when_native_unreadable(void) +{ + struct cmt *decoded; + char *buffer; + uint32_t flags = 0; + + decoded = relay_unreadable(CMT_FALSE, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 1); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + + decoded = relay_unreadable(CMT_TRUE, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); +} + +/* A released relay changes the value of series "b" without changing its + * time. The metadata recorded for the old value must not describe the new + * one, from the sidecar or from the legacy tree. */ +static void test_otlp_restore_drops_changed_value(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 1, 1, CMT_FALSE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + TEST_ASSERT(cmt_gauge_set(gauge, 100, 42, 1, (char *[]) {"b"}) == 0); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 1); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* Two windows of one series differ only by start time. A released relay + * drops the first: its metadata must not move to the survivor. */ +static void test_otlp_restore_binds_points_by_start_time(void) +{ + struct cmt *ctx; + struct cmt *window; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_metric *first; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + window = relay_window(10, 1, 1.0); + TEST_ASSERT(cmt_cat(ctx, window) == 0); + cmt_destroy(window); + window = relay_window(20, 0, 2.0); + TEST_ASSERT(cmt_cat(ctx, window) == 0); + cmt_destroy(window); + + gauge = cfl_list_entry_first(&ctx->gauges, struct cmt_gauge, _head); + TEST_ASSERT(cfl_list_size(&gauge->map->metrics) == 2); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + + relay_install(ctx, gauge->map, internal, external); + first = cfl_list_entry_first(&gauge->map->metrics, struct cmt_metric, _head); + TEST_ASSERT(cmt_metric_get_start_timestamp(first) == 10); + cmt_map_metric_destroy(first); + + decoded = relay_round_trip(ctx, &buffer); + TEST_CHECK(relay_otlp_flags(decoded, "a", &flags) && flags == 0); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + +/* A v2.3.0 sidecar binds by position. When a released relay removed a series, + * its points are not restored, and the legacy tree must still provide what + * released readers provide, even though family metadata was restored. */ +static void test_otlp_restore_keeps_unbound_legacy_points(void) +{ + struct cmt *ctx; + struct cmt *decoded; + struct cmt_gauge *gauge; + struct cmt_gauge *result; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + char *buffer; + uint32_t flags; + + ctx = cmt_create(); + TEST_ASSERT(ctx != NULL); + gauge = relay_gauge(ctx, 0, 1, CMT_TRUE); + internal = relay_snapshot(ctx, CMT_TRUE); + external = relay_snapshot(ctx, CMT_FALSE); + TEST_ASSERT(internal != NULL && external != NULL); + relay_strip_point_ids(internal); + + relay_install(ctx, gauge->map, internal, external); + cmt_map_metric_destroy(cmt_map_metric_get(&gauge->opts, gauge->map, 1, + (char *[]) {"a"}, CMT_FALSE)); + + decoded = relay_round_trip(ctx, &buffer); + result = cfl_list_entry_first(&decoded->gauges, struct cmt_gauge, _head); + TEST_CHECK(result->map->otlp_metadata != NULL); + TEST_CHECK(relay_otlp_flags(decoded, "b", &flags) && flags == 1); + cmt_decode_msgpack_destroy(decoded); + cmt_encode_msgpack_destroy(buffer); + cmt_destroy(ctx); +} + TEST_LIST = { + {"otlp_restore_binds_points_by_identity", test_otlp_restore_binds_points_by_identity}, + {"otlp_restore_drops_stale_observation", test_otlp_restore_drops_stale_observation}, + {"otlp_restore_drops_changed_value", test_otlp_restore_drops_changed_value}, + {"otlp_restore_keeps_legacy_when_native_unreadable", test_otlp_restore_keeps_legacy_when_native_unreadable}, + {"otlp_restore_drops_shared_legacy_key", test_otlp_restore_drops_shared_legacy_key}, + {"otlp_restore_binds_points_by_start_time", test_otlp_restore_binds_points_by_start_time}, + {"otlp_restore_binds_points_by_label_values", test_otlp_restore_binds_points_by_label_values}, + {"otlp_restore_binds_reordered_points", test_otlp_restore_binds_reordered_points}, + {"otlp_restore_keeps_unbound_legacy_points", test_otlp_restore_keeps_unbound_legacy_points}, + {"otlp_nested_empty_attributes_keep_point", test_otlp_nested_empty_attributes_keep_point}, + {"otlp_restore_preserves_legacy_only_family", test_otlp_restore_preserves_legacy_only_family}, {"otlp_unrepresentable_attribute_keeps_point", test_otlp_unrepresentable_attribute_keeps_point}, {"otlp_metadata_legacy_external_tree", test_otlp_metadata_legacy_external_tree}, {"otlp_metadata_deep_nesting", test_otlp_metadata_deep_nesting}, diff --git a/tests/otlp_pipeline.c b/tests/otlp_pipeline.c new file mode 100644 index 00000000..0dba4c2c --- /dev/null +++ b/tests/otlp_pipeline.c @@ -0,0 +1,1794 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * OTLP traffic through the transformations Fluent Bit applies to it: + * + * direct : in_opentelemetry -> out_opentelemetry in one process + * chunk : OTLP -> MessagePack chunk -> OTLP (storage, output processors) + * forward : two MessagePack hops (agent -> forward -> aggregator -> OTLP) + * cat : chunk decoded and merged with cmt_cat (batching, processors) + * + * Every data point of the input request is reduced to a canonical line that + * includes its resource, scope, metric and point fields with exact bit + * patterns for doubles and typed, order independent attributes. A pipeline is + * lossless when the output request produces the same multiset of lines. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "cmt_tests.h" + +/* Builder and canonicalization allocations: abort rather than flood the + * check log, these are test infrastructure, not behavior under test. */ +static void *require(void *pointer) +{ + if (pointer == NULL) { + abort(); + } + return pointer; +} + +typedef Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest otlp_request; +typedef Opentelemetry__Proto__Metrics__V1__ResourceMetrics otlp_resource_metrics; +typedef Opentelemetry__Proto__Metrics__V1__ScopeMetrics otlp_scope_metrics; +typedef Opentelemetry__Proto__Metrics__V1__Metric otlp_metric; +typedef Opentelemetry__Proto__Metrics__V1__NumberDataPoint otlp_number_point; +typedef Opentelemetry__Proto__Metrics__V1__HistogramDataPoint otlp_histogram_point; +typedef Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint otlp_exp_point; +typedef Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets otlp_exp_buckets; +typedef Opentelemetry__Proto__Metrics__V1__SummaryDataPoint otlp_summary_point; +typedef Opentelemetry__Proto__Metrics__V1__SummaryDataPoint__ValueAtQuantile otlp_quantile; +typedef Opentelemetry__Proto__Metrics__V1__Exemplar otlp_exemplar; +typedef Opentelemetry__Proto__Common__V1__KeyValue otlp_kv; +typedef Opentelemetry__Proto__Common__V1__AnyValue otlp_any; + +#define TEMPORALITY_DELTA \ + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_DELTA +#define TEMPORALITY_CUMULATIVE \ + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE + +enum pipeline { + PIPELINE_DIRECT = 0, + PIPELINE_CHUNK, + PIPELINE_FORWARD, + PIPELINE_CAT, + PIPELINE_COUNT +}; + +static const char *pipeline_names[] = {"direct", "chunk", "forward", "cat"}; + +/* Every builder allocation is released at once after a case. */ +static void **allocations; +static size_t allocation_count; +static size_t allocation_size; + +static void *test_alloc(size_t size) +{ + void **grown; + void *memory; + + if (allocation_count == allocation_size) { + allocation_size = allocation_size ? allocation_size * 2 : 64; + grown = require(realloc(allocations, allocation_size * sizeof(*allocations))); + allocations = grown; + } + memory = require(calloc(1, size ? size : 1)); + allocations[allocation_count++] = memory; + return memory; +} + +static void test_alloc_release(void) +{ + size_t index; + + for (index = 0; index < allocation_count; index++) { + free(allocations[index]); + } + free(allocations); + allocations = NULL; + allocation_count = 0; + allocation_size = 0; +} + +static void **pointer_list(size_t count, va_list arguments) +{ + void **list; + size_t index; + + list = test_alloc(sizeof(void *) * count); + for (index = 0; index < count; index++) { + list[index] = va_arg(arguments, void *); + } + return list; +} + +/* AnyValue and KeyValue builders */ + +static otlp_any *any_string(const char *value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE; + any->string_value = (char *) value; + return any; +} + +static otlp_any *any_int(int64_t value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE; + any->int_value = value; + return any; +} + +static otlp_any *any_double(double value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE; + any->double_value = value; + return any; +} + +static otlp_any *any_bool(int value) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE; + any->bool_value = value; + return any; +} + +static otlp_any *any_bytes(const char *value, size_t length) +{ + otlp_any *any = test_alloc(sizeof(*any)); + + opentelemetry__proto__common__v1__any_value__init(any); + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE; + any->bytes_value.data = (uint8_t *) value; + any->bytes_value.len = length; + return any; +} + +static otlp_any *any_array(size_t count, ...) +{ + va_list arguments; + otlp_any *any = test_alloc(sizeof(*any)); + Opentelemetry__Proto__Common__V1__ArrayValue *array = test_alloc(sizeof(*array)); + + opentelemetry__proto__common__v1__any_value__init(any); + opentelemetry__proto__common__v1__array_value__init(array); + va_start(arguments, count); + array->values = (otlp_any **) pointer_list(count, arguments); + va_end(arguments); + array->n_values = count; + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE; + any->array_value = array; + return any; +} + +static otlp_any *any_kvlist(size_t count, ...) +{ + va_list arguments; + otlp_any *any = test_alloc(sizeof(*any)); + Opentelemetry__Proto__Common__V1__KeyValueList *list = test_alloc(sizeof(*list)); + + opentelemetry__proto__common__v1__any_value__init(any); + opentelemetry__proto__common__v1__key_value_list__init(list); + va_start(arguments, count); + list->values = (otlp_kv **) pointer_list(count, arguments); + va_end(arguments); + list->n_values = count; + any->value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE; + any->kvlist_value = list; + return any; +} + +static otlp_kv *kv(const char *key, otlp_any *value) +{ + otlp_kv *pair = test_alloc(sizeof(*pair)); + + opentelemetry__proto__common__v1__key_value__init(pair); + pair->key = (char *) key; + pair->value = value; + return pair; +} + +static otlp_kv **kv_list(size_t count, ...) +{ + va_list arguments; + otlp_kv **list; + + va_start(arguments, count); + list = (otlp_kv **) pointer_list(count, arguments); + va_end(arguments); + return list; +} + +/* Data point builders */ + +static otlp_number_point *number_int(uint64_t start, uint64_t time, int64_t value, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_number_point *point = test_alloc(sizeof(*point)); + + opentelemetry__proto__metrics__v1__number_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + point->as_int = value; + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +static otlp_number_point *number_double(uint64_t start, uint64_t time, double value, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_number_point *point = number_int(start, time, 0, attribute_count, attributes); + + point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + point->as_double = value; + return point; +} + +static otlp_exemplar *exemplar_double(uint64_t time, double value, const char *trace_id, + const char *span_id, size_t attribute_count, + otlp_kv **attributes) +{ + otlp_exemplar *exemplar = test_alloc(sizeof(*exemplar)); + + opentelemetry__proto__metrics__v1__exemplar__init(exemplar); + exemplar->time_unix_nano = time; + exemplar->value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_DOUBLE; + exemplar->as_double = value; + if (trace_id != NULL) { + exemplar->trace_id.data = (uint8_t *) trace_id; + exemplar->trace_id.len = 16; + } + if (span_id != NULL) { + exemplar->span_id.data = (uint8_t *) span_id; + exemplar->span_id.len = 8; + } + exemplar->n_filtered_attributes = attribute_count; + exemplar->filtered_attributes = attributes; + return exemplar; +} + +static otlp_exemplar *exemplar_int(uint64_t time, int64_t value) +{ + otlp_exemplar *exemplar = exemplar_double(time, 0, NULL, NULL, 0, NULL); + + exemplar->value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_INT; + exemplar->as_int = value; + return exemplar; +} + +static otlp_exemplar **exemplar_list(size_t count, ...) +{ + va_list arguments; + otlp_exemplar **list; + + va_start(arguments, count); + list = (otlp_exemplar **) pointer_list(count, arguments); + va_end(arguments); + return list; +} + +static otlp_histogram_point *histogram_point(uint64_t start, uint64_t time, + size_t bound_count, const double *bounds, + const uint64_t *counts, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_histogram_point *point = test_alloc(sizeof(*point)); + size_t index; + + opentelemetry__proto__metrics__v1__histogram_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->n_explicit_bounds = bound_count; + point->explicit_bounds = test_alloc(sizeof(double) * (bound_count + 1)); + if (bound_count > 0) { + memcpy(point->explicit_bounds, bounds, sizeof(double) * bound_count); + } + point->n_bucket_counts = bound_count + 1; + point->bucket_counts = test_alloc(sizeof(uint64_t) * (bound_count + 1)); + memcpy(point->bucket_counts, counts, sizeof(uint64_t) * (bound_count + 1)); + for (index = 0; index <= bound_count; index++) { + point->count += counts[index]; + } + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +static otlp_exp_buckets *exp_buckets(int32_t offset, size_t count, const uint64_t *counts) +{ + otlp_exp_buckets *buckets = test_alloc(sizeof(*buckets)); + + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(buckets); + buckets->offset = offset; + buckets->n_bucket_counts = count; + buckets->bucket_counts = test_alloc(sizeof(uint64_t) * (count + 1)); + memcpy(buckets->bucket_counts, counts, sizeof(uint64_t) * count); + return buckets; +} + +static otlp_exp_point *exp_point(uint64_t start, uint64_t time, int32_t scale, + uint64_t zero_count, double zero_threshold, + otlp_exp_buckets *positive, otlp_exp_buckets *negative, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_exp_point *point = test_alloc(sizeof(*point)); + size_t index; + + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->scale = scale; + point->zero_count = zero_count; + point->zero_threshold = zero_threshold; + point->positive = positive; + point->negative = negative; + point->count = zero_count; + for (index = 0; positive != NULL && index < positive->n_bucket_counts; index++) { + point->count += positive->bucket_counts[index]; + } + for (index = 0; negative != NULL && index < negative->n_bucket_counts; index++) { + point->count += negative->bucket_counts[index]; + } + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +static otlp_summary_point *summary_point(uint64_t start, uint64_t time, uint64_t count, + double sum, size_t quantile_count, + const double *quantiles, const double *values, + size_t attribute_count, otlp_kv **attributes) +{ + otlp_summary_point *point = test_alloc(sizeof(*point)); + size_t index; + + opentelemetry__proto__metrics__v1__summary_data_point__init(point); + point->start_time_unix_nano = start; + point->time_unix_nano = time; + point->count = count; + point->sum = sum; + point->n_quantile_values = quantile_count; + point->quantile_values = test_alloc(sizeof(otlp_quantile *) * (quantile_count + 1)); + for (index = 0; index < quantile_count; index++) { + point->quantile_values[index] = test_alloc(sizeof(otlp_quantile)); + opentelemetry__proto__metrics__v1__summary_data_point__value_at_quantile__init( + point->quantile_values[index]); + point->quantile_values[index]->quantile = quantiles[index]; + point->quantile_values[index]->value = values[index]; + } + point->n_attributes = attribute_count; + point->attributes = attributes; + return point; +} + +/* Metric builders: points are passed as a count and a variadic list. */ + +static otlp_metric *metric_base(const char *name, const char *description, const char *unit) +{ + otlp_metric *metric = test_alloc(sizeof(*metric)); + + opentelemetry__proto__metrics__v1__metric__init(metric); + metric->name = (char *) name; + metric->description = (char *) (description ? description : ""); + metric->unit = (char *) (unit ? unit : ""); + return metric; +} + +static otlp_metric *metric_gauge(const char *name, const char *description, const char *unit, + size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, description, unit); + Opentelemetry__Proto__Metrics__V1__Gauge *gauge = test_alloc(sizeof(*gauge)); + + opentelemetry__proto__metrics__v1__gauge__init(gauge); + va_start(arguments, count); + gauge->data_points = (otlp_number_point **) pointer_list(count, arguments); + va_end(arguments); + gauge->n_data_points = count; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE; + metric->gauge = gauge; + return metric; +} + +static otlp_metric *metric_sum(const char *name, const char *description, const char *unit, + int temporality, int monotonic, size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, description, unit); + Opentelemetry__Proto__Metrics__V1__Sum *sum = test_alloc(sizeof(*sum)); + + opentelemetry__proto__metrics__v1__sum__init(sum); + va_start(arguments, count); + sum->data_points = (otlp_number_point **) pointer_list(count, arguments); + va_end(arguments); + sum->n_data_points = count; + sum->aggregation_temporality = temporality; + sum->is_monotonic = monotonic; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM; + metric->sum = sum; + return metric; +} + +static otlp_metric *metric_histogram(const char *name, const char *unit, int temporality, + size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, "histogram", unit); + Opentelemetry__Proto__Metrics__V1__Histogram *histogram = test_alloc(sizeof(*histogram)); + + opentelemetry__proto__metrics__v1__histogram__init(histogram); + va_start(arguments, count); + histogram->data_points = (otlp_histogram_point **) pointer_list(count, arguments); + va_end(arguments); + histogram->n_data_points = count; + histogram->aggregation_temporality = temporality; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM; + metric->histogram = histogram; + return metric; +} + +static otlp_metric *metric_exp_histogram(const char *name, int temporality, size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, "exponential histogram", "s"); + Opentelemetry__Proto__Metrics__V1__ExponentialHistogram *histogram = + test_alloc(sizeof(*histogram)); + + opentelemetry__proto__metrics__v1__exponential_histogram__init(histogram); + va_start(arguments, count); + histogram->data_points = (otlp_exp_point **) pointer_list(count, arguments); + va_end(arguments); + histogram->n_data_points = count; + histogram->aggregation_temporality = temporality; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM; + metric->exponential_histogram = histogram; + return metric; +} + +static otlp_metric *metric_summary(const char *name, size_t count, ...) +{ + va_list arguments; + otlp_metric *metric = metric_base(name, "summary", "ms"); + Opentelemetry__Proto__Metrics__V1__Summary *summary = test_alloc(sizeof(*summary)); + + opentelemetry__proto__metrics__v1__summary__init(summary); + va_start(arguments, count); + summary->data_points = (otlp_summary_point **) pointer_list(count, arguments); + va_end(arguments); + summary->n_data_points = count; + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUMMARY; + metric->summary = summary; + return metric; +} + +static otlp_scope_metrics *scope(const char *name, const char *version, + size_t attribute_count, otlp_kv **attributes, + size_t count, ...) +{ + va_list arguments; + otlp_scope_metrics *scope_metrics = test_alloc(sizeof(*scope_metrics)); + Opentelemetry__Proto__Common__V1__InstrumentationScope *instrumentation; + + opentelemetry__proto__metrics__v1__scope_metrics__init(scope_metrics); + if (name != NULL) { + instrumentation = test_alloc(sizeof(*instrumentation)); + opentelemetry__proto__common__v1__instrumentation_scope__init(instrumentation); + instrumentation->name = (char *) name; + instrumentation->version = (char *) (version ? version : ""); + instrumentation->n_attributes = attribute_count; + instrumentation->attributes = attributes; + scope_metrics->scope = instrumentation; + } + va_start(arguments, count); + scope_metrics->metrics = (otlp_metric **) pointer_list(count, arguments); + va_end(arguments); + scope_metrics->n_metrics = count; + return scope_metrics; +} + +static otlp_resource_metrics *resource(size_t attribute_count, otlp_kv **attributes, + size_t count, ...) +{ + va_list arguments; + otlp_resource_metrics *resource_metrics = test_alloc(sizeof(*resource_metrics)); + Opentelemetry__Proto__Resource__V1__Resource *resource_value = + test_alloc(sizeof(*resource_value)); + + opentelemetry__proto__metrics__v1__resource_metrics__init(resource_metrics); + opentelemetry__proto__resource__v1__resource__init(resource_value); + resource_value->n_attributes = attribute_count; + resource_value->attributes = attributes; + resource_metrics->resource = resource_value; + va_start(arguments, count); + resource_metrics->scope_metrics = (otlp_scope_metrics **) pointer_list(count, arguments); + va_end(arguments); + resource_metrics->n_scope_metrics = count; + return resource_metrics; +} + +static otlp_request *request(size_t count, ...) +{ + va_list arguments; + otlp_request *value = test_alloc(sizeof(*value)); + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(value); + va_start(arguments, count); + value->resource_metrics = (otlp_resource_metrics **) pointer_list(count, arguments); + va_end(arguments); + value->n_resource_metrics = count; + return value; +} + +/* Canonical form */ + +/* cmt_cat merges metric families only; context metadata (resource, scope) + * is not merged by design, so the cat pipeline compares families alone. */ +static int canon_include_context = CMT_TRUE; + +struct lines { + cfl_sds_t *entries; + size_t count; + size_t size; +}; + +static void lines_add(struct lines *lines, cfl_sds_t line) +{ + cfl_sds_t *grown; + + if (lines->count == lines->size) { + lines->size = lines->size ? lines->size * 2 : 16; + grown = require(realloc(lines->entries, lines->size * sizeof(*grown))); + lines->entries = grown; + } + lines->entries[lines->count++] = line; +} + +static void lines_destroy(struct lines *lines) +{ + size_t index; + + for (index = 0; index < lines->count; index++) { + cfl_sds_destroy(lines->entries[index]); + } + free(lines->entries); + memset(lines, 0, sizeof(*lines)); +} + +static cfl_sds_t sds_printf(cfl_sds_t buffer, const char *format, ...) +{ + va_list arguments; + char stack[512]; + int length; + + va_start(arguments, format); + length = vsnprintf(stack, sizeof(stack), format, arguments); + va_end(arguments); + if (length < 0 || (size_t) length >= sizeof(stack)) { + abort(); + } + return require(cfl_sds_cat(buffer, stack, length)); +} + +static cfl_sds_t canon_double(cfl_sds_t buffer, double value) +{ + uint64_t bits; + + memcpy(&bits, &value, sizeof(bits)); + return sds_printf(buffer, "%016" PRIx64, bits); +} + +static cfl_sds_t canon_bytes(cfl_sds_t buffer, const uint8_t *data, size_t length) +{ + size_t index; + + for (index = 0; index < length; index++) { + buffer = sds_printf(buffer, "%02x", data[index]); + } + return buffer; +} + +static int compare_strings(const void *left, const void *right) +{ + return strcmp(*(const char **) left, *(const char **) right); +} + +static cfl_sds_t canon_attributes(cfl_sds_t buffer, size_t count, otlp_kv **attributes); + +static cfl_sds_t canon_any(cfl_sds_t buffer, otlp_any *value) +{ + size_t index; + + if (value == NULL) { + return cfl_sds_cat(buffer, "none", 4); + } + switch (value->value_case) { + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: + buffer = cfl_sds_cat(buffer, "s:", 2); + return cfl_sds_cat(buffer, value->string_value, strlen(value->string_value)); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: + return sds_printf(buffer, "i:%" PRId64, value->int_value); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: + buffer = cfl_sds_cat(buffer, "d:", 2); + return canon_double(buffer, value->double_value); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: + return sds_printf(buffer, "b:%d", value->bool_value ? 1 : 0); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: + buffer = cfl_sds_cat(buffer, "x:", 2); + return canon_bytes(buffer, value->bytes_value.data, value->bytes_value.len); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: + buffer = cfl_sds_cat(buffer, "[", 1); + for (index = 0; index < value->array_value->n_values; index++) { + buffer = canon_any(buffer, value->array_value->values[index]); + buffer = cfl_sds_cat(buffer, ",", 1); + } + return cfl_sds_cat(buffer, "]", 1); + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: + return canon_attributes(buffer, value->kvlist_value->n_values, + value->kvlist_value->values); + default: + return cfl_sds_cat(buffer, "unset", 5); + } +} + +/* Attributes are a set in OTLP: canonicalize independently of order. */ +static cfl_sds_t canon_attributes(cfl_sds_t buffer, size_t count, otlp_kv **attributes) +{ + cfl_sds_t *entries; + size_t index; + + entries = require(calloc(count + 1, sizeof(*entries))); + for (index = 0; index < count; index++) { + entries[index] = cfl_sds_create(attributes[index]->key); + entries[index] = cfl_sds_cat(entries[index], "=", 1); + entries[index] = canon_any(entries[index], attributes[index]->value); + } + qsort(entries, count, sizeof(*entries), compare_strings); + buffer = cfl_sds_cat(buffer, "{", 1); + for (index = 0; index < count; index++) { + buffer = cfl_sds_cat(buffer, entries[index], cfl_sds_len(entries[index])); + buffer = cfl_sds_cat(buffer, ";", 1); + cfl_sds_destroy(entries[index]); + } + free(entries); + return cfl_sds_cat(buffer, "}", 1); +} + +static cfl_sds_t canon_exemplars(cfl_sds_t buffer, size_t count, otlp_exemplar **exemplars) +{ + size_t index; + otlp_exemplar *exemplar; + + buffer = cfl_sds_cat(buffer, " ex=[", 5); + for (index = 0; index < count; index++) { + exemplar = exemplars[index]; + buffer = sds_printf(buffer, "t%" PRIu64 ",", exemplar->time_unix_nano); + if (exemplar->value_case == OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_INT) { + buffer = sds_printf(buffer, "i%" PRId64, exemplar->as_int); + } + else { + buffer = cfl_sds_cat(buffer, "d", 1); + buffer = canon_double(buffer, exemplar->as_double); + } + buffer = cfl_sds_cat(buffer, ",tr", 3); + buffer = canon_bytes(buffer, exemplar->trace_id.data, exemplar->trace_id.len); + buffer = cfl_sds_cat(buffer, ",sp", 3); + buffer = canon_bytes(buffer, exemplar->span_id.data, exemplar->span_id.len); + buffer = canon_attributes(buffer, exemplar->n_filtered_attributes, + exemplar->filtered_attributes); + buffer = cfl_sds_cat(buffer, ";", 1); + } + return cfl_sds_cat(buffer, "]", 1); +} + +static cfl_sds_t canon_point_common(cfl_sds_t buffer, size_t attribute_count, + otlp_kv **attributes, uint64_t start, uint64_t time, + uint32_t flags) +{ + buffer = cfl_sds_cat(buffer, " attrs=", 7); + buffer = canon_attributes(buffer, attribute_count, attributes); + return sds_printf(buffer, " st=%" PRIu64 " t=%" PRIu64 " flags=%u", start, time, flags); +} + +static cfl_sds_t canon_optional_double(cfl_sds_t buffer, const char *name, + int present, double value) +{ + buffer = sds_printf(buffer, " %s=", name); + if (!present) { + return cfl_sds_cat(buffer, "-", 1); + } + return canon_double(buffer, value); +} + +static void canon_metric(struct lines *lines, cfl_sds_t prefix, otlp_metric *metric) +{ + size_t index; + size_t bucket; + cfl_sds_t line; + otlp_number_point **numbers = NULL; + size_t number_count = 0; + otlp_number_point *number; + otlp_histogram_point *histogram; + otlp_exp_point *exp; + otlp_summary_point *summary; + + prefix = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + prefix = sds_printf(prefix, "|metric name=%s desc=%s unit=%s meta=", + metric->name, metric->description, metric->unit); + prefix = canon_attributes(prefix, metric->n_metadata, metric->metadata); + + switch (metric->data_case) { + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE: + prefix = cfl_sds_cat(prefix, " gauge", 6); + numbers = metric->gauge->data_points; + number_count = metric->gauge->n_data_points; + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM: + prefix = sds_printf(prefix, " sum temporality=%d monotonic=%d", + metric->sum->aggregation_temporality, + metric->sum->is_monotonic ? 1 : 0); + numbers = metric->sum->data_points; + number_count = metric->sum->n_data_points; + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM: + prefix = sds_printf(prefix, " histogram temporality=%d", + metric->histogram->aggregation_temporality); + for (index = 0; index < metric->histogram->n_data_points; index++) { + histogram = metric->histogram->data_points[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, histogram->n_attributes, histogram->attributes, + histogram->start_time_unix_nano, + histogram->time_unix_nano, histogram->flags); + line = sds_printf(line, " count=%" PRIu64 " bounds=", histogram->count); + for (bucket = 0; bucket < histogram->n_explicit_bounds; bucket++) { + line = canon_double(line, histogram->explicit_bounds[bucket]); + line = cfl_sds_cat(line, ",", 1); + } + line = cfl_sds_cat(line, " buckets=", 9); + for (bucket = 0; bucket < histogram->n_bucket_counts; bucket++) { + line = sds_printf(line, "%" PRIu64 ",", histogram->bucket_counts[bucket]); + } + line = canon_optional_double(line, "sum", histogram->has_sum, histogram->sum); + line = canon_optional_double(line, "min", histogram->has_min, histogram->min); + line = canon_optional_double(line, "max", histogram->has_max, histogram->max); + line = canon_exemplars(line, histogram->n_exemplars, histogram->exemplars); + lines_add(lines, line); + } + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM: + prefix = sds_printf(prefix, " exp_histogram temporality=%d", + metric->exponential_histogram->aggregation_temporality); + for (index = 0; index < metric->exponential_histogram->n_data_points; index++) { + exp = metric->exponential_histogram->data_points[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, exp->n_attributes, exp->attributes, + exp->start_time_unix_nano, exp->time_unix_nano, + exp->flags); + line = sds_printf(line, " count=%" PRIu64 " scale=%d zero=%" PRIu64 " zt=", + exp->count, exp->scale, exp->zero_count); + line = canon_double(line, exp->zero_threshold); + /* Absent and empty bucket ranges carry the same information. */ + if (exp->positive != NULL && exp->positive->n_bucket_counts > 0) { + line = sds_printf(line, " pos@%d:", exp->positive->offset); + for (bucket = 0; bucket < exp->positive->n_bucket_counts; bucket++) { + line = sds_printf(line, "%" PRIu64 ",", exp->positive->bucket_counts[bucket]); + } + } + if (exp->negative != NULL && exp->negative->n_bucket_counts > 0) { + line = sds_printf(line, " neg@%d:", exp->negative->offset); + for (bucket = 0; bucket < exp->negative->n_bucket_counts; bucket++) { + line = sds_printf(line, "%" PRIu64 ",", exp->negative->bucket_counts[bucket]); + } + } + line = canon_optional_double(line, "sum", exp->has_sum, exp->sum); + line = canon_optional_double(line, "min", exp->has_min, exp->min); + line = canon_optional_double(line, "max", exp->has_max, exp->max); + line = canon_exemplars(line, exp->n_exemplars, exp->exemplars); + lines_add(lines, line); + } + break; + case OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUMMARY: + prefix = cfl_sds_cat(prefix, " summary", 8); + for (index = 0; index < metric->summary->n_data_points; index++) { + summary = metric->summary->data_points[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, summary->n_attributes, summary->attributes, + summary->start_time_unix_nano, + summary->time_unix_nano, summary->flags); + line = sds_printf(line, " count=%" PRIu64 " sum=", summary->count); + line = canon_double(line, summary->sum); + line = cfl_sds_cat(line, " q=", 3); + for (bucket = 0; bucket < summary->n_quantile_values; bucket++) { + line = canon_double(line, summary->quantile_values[bucket]->quantile); + line = cfl_sds_cat(line, ":", 1); + line = canon_double(line, summary->quantile_values[bucket]->value); + line = cfl_sds_cat(line, ",", 1); + } + lines_add(lines, line); + } + break; + default: + prefix = cfl_sds_cat(prefix, " unset", 6); + break; + } + + for (index = 0; index < number_count; index++) { + number = numbers[index]; + line = cfl_sds_create_len(prefix, cfl_sds_len(prefix)); + line = canon_point_common(line, number->n_attributes, number->attributes, + number->start_time_unix_nano, number->time_unix_nano, + number->flags); + if (number->value_case == OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT) { + line = sds_printf(line, " int=%" PRId64, number->as_int); + } + else { + line = cfl_sds_cat(line, " double=", 8); + line = canon_double(line, number->as_double); + } + line = canon_exemplars(line, number->n_exemplars, number->exemplars); + lines_add(lines, line); + } + cfl_sds_destroy(prefix); +} + +static void canon_request(struct lines *lines, otlp_request *value) +{ + size_t resource_index; + size_t scope_index; + size_t metric_index; + otlp_resource_metrics *resource_metrics; + otlp_scope_metrics *scope_metrics; + cfl_sds_t resource_prefix; + cfl_sds_t scope_prefix; + + for (resource_index = 0; resource_index < value->n_resource_metrics; resource_index++) { + resource_metrics = value->resource_metrics[resource_index]; + resource_prefix = cfl_sds_create("resource="); + if (!canon_include_context) { + resource_prefix = cfl_sds_cat(resource_prefix, "*", 1); + } + else if (resource_metrics->resource != NULL) { + resource_prefix = canon_attributes(resource_prefix, + resource_metrics->resource->n_attributes, + resource_metrics->resource->attributes); + resource_prefix = sds_printf(resource_prefix, " dropped=%u", + resource_metrics->resource->dropped_attributes_count); + } + resource_prefix = sds_printf(resource_prefix, " schema=%s", + !canon_include_context ? "*" : + resource_metrics->schema_url ? + resource_metrics->schema_url : ""); + for (scope_index = 0; scope_index < resource_metrics->n_scope_metrics; scope_index++) { + scope_metrics = resource_metrics->scope_metrics[scope_index]; + scope_prefix = cfl_sds_create_len(resource_prefix, cfl_sds_len(resource_prefix)); + scope_prefix = cfl_sds_cat(scope_prefix, "|scope=", 7); + if (!canon_include_context) { + scope_prefix = cfl_sds_cat(scope_prefix, "*", 1); + } + else if (scope_metrics->scope != NULL) { + scope_prefix = sds_printf(scope_prefix, "%s@%s dropped=%u ", + scope_metrics->scope->name, + scope_metrics->scope->version, + scope_metrics->scope->dropped_attributes_count); + scope_prefix = canon_attributes(scope_prefix, + scope_metrics->scope->n_attributes, + scope_metrics->scope->attributes); + } + scope_prefix = sds_printf(scope_prefix, " schema=%s", + !canon_include_context ? "*" : + scope_metrics->schema_url ? scope_metrics->schema_url : ""); + for (metric_index = 0; metric_index < scope_metrics->n_metrics; metric_index++) { + canon_metric(lines, scope_prefix, scope_metrics->metrics[metric_index]); + } + cfl_sds_destroy(scope_prefix); + } + cfl_sds_destroy(resource_prefix); + } +} + +/* Pipelines */ + +static struct cmt *msgpack_hop(struct cmt *context, int destroy_source) +{ + struct cmt *decoded = NULL; + char *buffer; + size_t size; + size_t offset = 0; + int result; + + result = cmt_encode_msgpack_create(context, &buffer, &size); + TEST_CHECK(result == 0); + if (result != 0) { + return NULL; + } + result = cmt_decode_msgpack_create(&decoded, buffer, size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(offset == size); + cmt_encode_msgpack_destroy(buffer); + if (destroy_source) { + cmt_decode_msgpack_destroy(context); + } + return result == CMT_DECODE_MSGPACK_SUCCESS ? decoded : NULL; +} + +/* Run one pipeline on a decoded context and append the OTLP output lines. */ +static int run_pipeline(struct lines *lines, struct cmt *context, int pipeline) +{ + struct cmt *current; + struct cmt *merged; + cfl_sds_t wire; + otlp_request *output; + + current = context; + merged = NULL; + if (pipeline != PIPELINE_DIRECT) { + current = msgpack_hop(context, CMT_FALSE); + if (current == NULL) { + return -1; + } + } + if (pipeline == PIPELINE_FORWARD) { + current = msgpack_hop(current, CMT_TRUE); + if (current == NULL) { + return -1; + } + } + if (pipeline == PIPELINE_CAT) { + merged = cmt_create(); + TEST_ASSERT(merged != NULL); + TEST_CHECK(cmt_cat(merged, current) == 0); + cmt_decode_msgpack_destroy(current); + current = merged; + } + + wire = cmt_encode_opentelemetry_create(current); + TEST_CHECK(wire != NULL); + if (wire != NULL) { + output = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(wire), (const uint8_t *) wire); + TEST_CHECK(output != NULL); + if (output != NULL) { + canon_request(lines, output); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked( + output, NULL); + } + cmt_encode_opentelemetry_destroy(wire); + } + + if (merged != NULL) { + cmt_destroy(merged); + } + else if (current != context) { + cmt_decode_msgpack_destroy(current); + } + return wire != NULL ? 0 : -1; +} + +static cfl_sds_t pack_request(otlp_request *value) +{ + size_t size; + uint8_t *buffer; + cfl_sds_t payload; + + size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(value); + buffer = require(calloc(1, size + 1)); + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(value, buffer); + payload = cfl_sds_create_len((char *) buffer, size); + free(buffer); + TEST_ASSERT(payload != NULL); + return payload; +} + +static void compare_lines(struct lines *expected, struct lines *actual, const char *pipeline) +{ + size_t index; + int equal; + + if (expected->count > 1) { + qsort(expected->entries, expected->count, sizeof(cfl_sds_t), compare_strings); + } + if (actual->count > 1) { + qsort(actual->entries, actual->count, sizeof(cfl_sds_t), compare_strings); + } + equal = expected->count == actual->count; + for (index = 0; equal && index < expected->count; index++) { + equal = strcmp(expected->entries[index], actual->entries[index]) == 0; + } + TEST_CHECK_(equal, "pipeline %s reproduces every data point", pipeline); + if (!equal) { + for (index = 0; index < expected->count; index++) { + TEST_MSG("expected: %s", expected->entries[index]); + } + for (index = 0; index < actual->count; index++) { + TEST_MSG("actual: %s", actual->entries[index]); + } + } +} + +/* + * Decode the request as in_opentelemetry does and check that every requested + * pipeline reproduces it. The canonical lines of 'expected' default to the + * input itself; a case passes its own expectation when a transformation is + * intentionally lossy. + */ +static void check_pipelines(otlp_request *input, otlp_request *expected, unsigned int pipelines) +{ + struct cfl_list contexts; + struct cfl_list *head; + struct lines expected_lines; + struct lines actual_lines; + struct cmt *context; + cfl_sds_t payload; + size_t offset; + int pipeline; + int result; + + payload = pack_request(input); + + for (pipeline = 0; pipeline < PIPELINE_COUNT; pipeline++) { + if ((pipelines & (1u << pipeline)) == 0) { + continue; + } + offset = 0; + result = cmt_decode_opentelemetry_create(&contexts, payload, cfl_sds_len(payload), + &offset); + TEST_CHECK_(result == CMT_DECODE_OPENTELEMETRY_SUCCESS, + "OTLP decode (result %d)", result); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + break; + } + canon_include_context = pipeline != PIPELINE_CAT; + memset(&expected_lines, 0, sizeof(expected_lines)); + memset(&actual_lines, 0, sizeof(actual_lines)); + canon_request(&expected_lines, expected != NULL ? expected : input); + cfl_list_foreach(head, &contexts) { + context = cfl_list_entry(head, struct cmt, _head); + TEST_CHECK(run_pipeline(&actual_lines, context, pipeline) == 0); + } + compare_lines(&expected_lines, &actual_lines, pipeline_names[pipeline]); + canon_include_context = CMT_TRUE; + lines_destroy(&expected_lines); + lines_destroy(&actual_lines); + cmt_decode_opentelemetry_destroy(&contexts); + } + + cfl_sds_destroy(payload); +} + +#define ALL_PIPELINES ((1u << PIPELINE_COUNT) - 1) + +/* Cases */ + +#define RESOURCE_ATTRIBUTES \ + 2, kv_list(2, kv("service.name", any_string("checkout")), \ + kv("host.name", any_string("node-1"))) + +static otlp_request *single_metric(otlp_metric *metric) +{ + return request(1, resource(RESOURCE_ATTRIBUTES, 1, + scope("io.opentelemetry.runtime", "1.2.0", 0, NULL, 1, metric))); +} + +static otlp_kv **attrs1(const char *key, otlp_any *value) +{ + return kv_list(1, kv(key, value)); +} + +static double double_from_bits(uint64_t bits) +{ + double value; + + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static void run_case(otlp_request *input) +{ + check_pipelines(input, NULL, ALL_PIPELINES); + test_alloc_release(); +} + +/* Counter: monotonic cumulative sum with integer and double points. */ +static void test_sum_monotonic_cumulative(void) +{ + run_case(single_metric( + metric_sum("http.server.requests", "requests", "{request}", TEMPORALITY_CUMULATIVE, 1, 2, + number_int(1000, 2000, 42, 1, attrs1("http.route", any_string("/a"))), + number_int(1000, 2000, 7, 1, attrs1("http.route", any_string("/b")))))); +} + +/* UpDownCounter: non-monotonic sums must not become monotonic counters. */ +static void test_sum_non_monotonic(void) +{ + run_case(single_metric( + metric_sum("queue.items", "items", "{item}", TEMPORALITY_CUMULATIVE, 0, 2, + number_int(1000, 2000, -5, 1, attrs1("queue", any_string("in"))), + number_double(1000, 2000, -0.25, 1, attrs1("queue", any_string("out")))))); +} + +/* SDKs configured for delta temporality (common with some backends). */ +static void test_sum_delta(void) +{ + run_case(single_metric( + metric_sum("bytes.sent", "bytes", "By", TEMPORALITY_DELTA, 1, 1, + number_int(1000, 2000, 512, 1, attrs1("peer", any_string("db")))))); +} + +static void test_gauge_integer_extremes(void) +{ + run_case(single_metric( + metric_gauge("extremes", "integer extremes", "1", 4, + number_int(0, 100, INT64_MIN, 1, attrs1("v", any_string("min"))), + number_int(0, 100, INT64_MAX, 1, attrs1("v", any_string("max"))), + number_int(0, 100, -1, 1, attrs1("v", any_string("minus_one"))), + number_int(0, 100, 9007199254740993LL, 1, attrs1("v", any_string("2^53+1")))))); +} + +/* Prometheus receivers emit staleness markers (0x7ff0000000000002) and + * applications report NaN, infinities and negative zero. */ +static void test_gauge_special_doubles(void) +{ + run_case(single_metric( + metric_gauge("specials", "special doubles", "1", 6, + number_double(0, 100, double_from_bits(0x7ff8000000000000ULL), 1, + attrs1("v", any_string("nan"))), + number_double(0, 100, double_from_bits(0x7ff0000000000002ULL), 1, + attrs1("v", any_string("stale"))), + number_double(0, 100, INFINITY, 1, attrs1("v", any_string("+inf"))), + number_double(0, 100, -INFINITY, 1, attrs1("v", any_string("-inf"))), + number_double(0, 100, -0.0, 1, attrs1("v", any_string("-0"))), + number_double(0, 100, 4.9406564584124654e-324, 1, attrs1("v", any_string("denormal")))))); +} + +static void test_point_flags_and_timestamps(void) +{ + otlp_number_point *missing; + + missing = number_double(10, 20, 0, 1, attrs1("v", any_string("no_value"))); + missing->flags = 1; + run_case(single_metric( + metric_gauge("timestamps", "flags and timestamps", "1", 3, + missing, + number_double(0, 9223372036854775813ULL, 1, 1, attrs1("v", any_string("late"))), + number_double(500, 400, 2, 1, attrs1("v", any_string("start_after_time")))))); +} + +static void test_histogram_explicit_bounds(void) +{ + static const double bounds[] = {0.005, 0.01, 0.1, 1, 10}; + static const uint64_t counts_a[] = {1, 2, 3, 4, 5, 6}; + static const uint64_t counts_b[] = {0, 0, 0, 0, 0, 9}; + otlp_histogram_point *a; + otlp_histogram_point *b; + + a = histogram_point(1000, 2000, 5, bounds, counts_a, 1, + attrs1("http.route", any_string("/a"))); + a->has_sum = 1; + a->sum = 123.5; + a->has_min = 1; + a->min = 0.001; + a->has_max = 1; + a->max = 42; + a->n_exemplars = 2; + a->exemplars = exemplar_list(2, + exemplar_double(1500, 0.007, "0123456789abcdef", "01234567", 1, + attrs1("user", any_string("u1"))), + exemplar_int(1600, 3)); + b = histogram_point(1000, 2000, 5, bounds, counts_b, 1, + attrs1("http.route", any_string("/b"))); + b->has_sum = 1; + b->sum = 900; + run_case(single_metric(metric_histogram("http.server.duration", "s", + TEMPORALITY_CUMULATIVE, 2, a, b))); +} + +static void test_histogram_delta_without_sum(void) +{ + static const double bounds[] = {1, 2}; + static const uint64_t counts[] = {3, 0, 1}; + + run_case(single_metric(metric_histogram("latency.delta", "ms", TEMPORALITY_DELTA, 1, + histogram_point(1000, 2000, 2, bounds, counts, 1, + attrs1("op", any_string("get")))))); +} + +/* + * A histogram with no explicit bounds has a single +Inf bucket (SDK views use + * it to keep only count and sum). CMetrics cannot represent it: the request is + * rejected. Accepting it would also emit a zero-bucket MessagePack histogram + * that every released reader rejects, so this is pinned until decided. + */ +static void test_histogram_without_bounds_is_rejected(void) +{ + static const uint64_t counts[] = {17}; + struct cfl_list contexts; + cfl_sds_t payload; + size_t offset = 0; + int result; + + payload = pack_request(single_metric(metric_histogram("single.bucket", "s", + TEMPORALITY_CUMULATIVE, 1, histogram_point(1000, 2000, 0, NULL, counts, 0, NULL)))); + result = cmt_decode_opentelemetry_create(&contexts, payload, cfl_sds_len(payload), &offset); + TEST_CHECK(result != CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cmt_decode_opentelemetry_destroy(&contexts); + } + cfl_sds_destroy(payload); + test_alloc_release(); +} + +static void test_exponential_histogram_layouts(void) +{ + static const uint64_t positive[] = {1, 0, 4, 2}; + static const uint64_t negative[] = {3, 1}; + static const uint64_t fine[] = {5}; + otlp_exp_point *coarse; + otlp_exp_point *precise; + otlp_exp_point *zeros; + + coarse = exp_point(1000, 2000, -3, 2, 1e-9, exp_buckets(-5, 4, positive), + exp_buckets(2, 2, negative), 1, attrs1("layout", any_string("coarse"))); + coarse->has_sum = 1; + coarse->sum = -12.5; + coarse->has_min = 1; + coarse->min = -40; + coarse->has_max = 1; + coarse->max = 300; + precise = exp_point(1000, 2000, 20, 0, 0, exp_buckets(1048576, 1, fine), NULL, 1, + attrs1("layout", any_string("precise"))); + precise->has_sum = 1; + precise->sum = 10; + zeros = exp_point(1000, 2000, 0, 6, 0.5, NULL, NULL, 1, + attrs1("layout", any_string("zeros"))); + run_case(single_metric(metric_exp_histogram("rpc.duration", TEMPORALITY_DELTA, 3, + coarse, precise, zeros))); +} + +static void test_summary_quantiles(void) +{ + static const double quantiles[] = {0, 0.5, 0.99, 1}; + static const double values[] = {0.1, 2.5, 9.75, 12}; + + run_case(single_metric(metric_summary("gc.pause", 2, + summary_point(1000, 2000, 40, 100.5, 4, quantiles, values, 1, + attrs1("gen", any_string("young"))), + summary_point(1000, 2000, 2, 30, 4, quantiles, values, 1, + attrs1("gen", any_string("old")))))); +} + +static void test_summary_without_quantiles(void) +{ + run_case(single_metric(metric_summary("requests.summary", 1, + summary_point(1000, 2000, 5, 12.5, 0, NULL, NULL, 1, + attrs1("route", any_string("/")))))); +} + +/* Attribute values of every OTLP type keep their type end to end. */ +static void test_typed_attributes(void) +{ + run_case(single_metric(metric_gauge("typed", "typed attributes", "1", 1, + number_double(0, 100, 1, 9, kv_list(9, + kv("string", any_string("value")), + kv("empty", any_string("")), + kv("unicode", any_string("caf\xc3\xa9 \xe2\x82\xac")), + kv("int", any_int(-42)), + kv("double", any_double(2.5)), + kv("bool", any_bool(1)), + kv("bytes", any_bytes("\x00\xff\x10", 3)), + kv("array", any_array(3, any_string("a"), any_int(1), any_bool(0))), + kv("map", any_kvlist(2, kv("inner", any_string("x")), kv("n", any_double(0.5))))))))); +} + +static otlp_request *type_identity_request(int collapsed) +{ + if (collapsed) { + return single_metric(metric_gauge("identity", "type identity", "1", 1, + number_double(0, 100, 3, 1, attrs1("k", any_double(7))))); + } + return single_metric(metric_gauge("identity", "type identity", "1", 3, + number_double(0, 100, 1, 1, attrs1("k", any_int(7))), + number_double(0, 100, 2, 1, attrs1("k", any_string("7"))), + number_double(0, 100, 3, 1, attrs1("k", any_double(7))))); +} + +/* + * int 7, string "7" and double 7 are three OTLP series and stay three through + * OTLP and MessagePack. cmt_cat matches series by their string labels, which + * are identical, so it merges them and the last point wins. Pinned: series + * identity in cmt_cat does not consider attribute types. + */ +static void test_attribute_type_identity(void) +{ + check_pipelines(type_identity_request(CMT_FALSE), NULL, + ALL_PIPELINES & ~(1u << PIPELINE_CAT)); + check_pipelines(type_identity_request(CMT_FALSE), type_identity_request(CMT_TRUE), + 1u << PIPELINE_CAT); + test_alloc_release(); +} + +/* Native metadata as a released relay carries it: opaque context metadata + * written back unchanged, whatever it does to the points. */ +static struct cfl_kvlist *relay_copy(struct cmt *context, int internal) +{ + struct cfl_kvlist *copy = NULL; + mpack_writer_t writer; + mpack_reader_t reader; + char *buffer = NULL; + size_t size = 0; + int result; + + mpack_writer_init_growable(&writer, &buffer, &size); + result = internal ? cmt_otlp_metadata_pack(&writer, context) : + cmt_otlp_metadata_pack_external(&writer, context); + TEST_ASSERT(mpack_writer_destroy(&writer) == mpack_ok && result == 0); + mpack_reader_init_data(&reader, buffer, size); + result = unpack_cfl_kvlist(&reader, ©); + TEST_ASSERT(mpack_reader_destroy(&reader) == mpack_ok && result == 0); + free(buffer); + return copy; +} + +/* Distribution points that differ only by attribute type, count and sum, + * but not by distribution, stay distinct through every hop. */ +static void test_distribution_type_identity(void) +{ + static const double bounds[] = {1, 2}; + static const uint64_t buckets_a[] = {1, 0, 1}; + static const uint64_t buckets_b[] = {0, 2, 0}; + static const double quantiles[] = {0.5}; + static const double quantile_a[] = {1}; + static const double quantile_b[] = {2}; + static const uint64_t positive_a[] = {1, 1}; + static const uint64_t positive_b[] = {2, 0}; + unsigned int mask = ALL_PIPELINES & ~(1u << PIPELINE_CAT); + otlp_histogram_point *histogram[2]; + otlp_summary_point *summary[2]; + otlp_exp_point *exp[2]; + int index; + + histogram[0] = histogram_point(0, 100, 2, bounds, buckets_a, 1, attrs1("k", any_int(7))); + histogram[1] = histogram_point(0, 100, 2, bounds, buckets_b, 1, attrs1("k", any_string("7"))); + summary[0] = summary_point(0, 100, 2, 3, 1, quantiles, quantile_a, 1, attrs1("k", any_int(7))); + summary[1] = summary_point(0, 100, 2, 3, 1, quantiles, quantile_b, 1, + attrs1("k", any_string("7"))); + exp[0] = exp_point(0, 100, 0, 0, 0, exp_buckets(0, 2, positive_a), NULL, 1, + attrs1("k", any_int(7))); + exp[1] = exp_point(0, 100, 0, 0, 0, exp_buckets(0, 2, positive_b), NULL, 1, + attrs1("k", any_string("7"))); + for (index = 0; index < 2; index++) { + histogram[index]->has_sum = 1; + histogram[index]->sum = 3; + exp[index]->has_sum = 1; + exp[index]->sum = 3; + } + histogram[0]->flags = 1; + summary[0]->flags = 1; + exp[0]->flags = 1; + + check_pipelines(single_metric(metric_histogram("latency", "s", TEMPORALITY_CUMULATIVE, 2, + histogram[0], histogram[1])), NULL, mask); + check_pipelines(single_metric(metric_summary("rpc", 2, summary[0], summary[1])), NULL, mask); + check_pipelines(single_metric(metric_exp_histogram("exp", TEMPORALITY_CUMULATIVE, 2, + exp[0], exp[1])), NULL, mask); + test_alloc_release(); +} + +struct typed_relay { + struct cfl_list contexts; + struct cmt *decoded; + char *buffer; + cfl_sds_t payload; + cfl_sds_t wire; + otlp_request *output; +}; + +/* Points of one series and time that differ only by attribute type: integer + * 7, string "7" and double 7.0. */ +static otlp_request *typed_points(const double *values, const uint32_t *flags) +{ + otlp_request *request; + otlp_metric *metric; + size_t index; + + request = single_metric(metric_gauge("identity", "type identity", "1", 3, + number_double(0, 100, values[0], 1, attrs1("k", any_int(7))), + number_double(0, 100, values[1], 1, attrs1("k", any_string("7"))), + number_double(0, 100, values[2], 1, attrs1("k", any_double(7))))); + metric = request->resource_metrics[0]->scope_metrics[0]->metrics[0]; + for (index = 0; index < 3; index++) { + metric->gauge->data_points[index]->flags = flags[index]; + } + return request; +} + +/* Decode the request, then act as a released relay: keep native metadata as + * opaque context metadata and either drop the point at drop_index or swap + * the first two points, then decode the relayed chunk and export it. */ +static void typed_relay_run(struct typed_relay *relay, otlp_request *input, + int drop_index, int swap) +{ + struct cfl_list *head; + struct cfl_list *first; + struct cmt *context; + struct cmt_gauge *gauge; + struct cmt_metric *sample; + struct cfl_kvlist *internal; + struct cfl_kvlist *external; + size_t size; + size_t offset = 0; + int index = 0; + + memset(relay, 0, sizeof(*relay)); + relay->payload = pack_request(input); + TEST_ASSERT(cmt_decode_opentelemetry_create(&relay->contexts, relay->payload, + cfl_sds_len(relay->payload), &offset) == + CMT_DECODE_OPENTELEMETRY_SUCCESS); + context = cfl_list_entry_first(&relay->contexts, struct cmt, _head); + gauge = cfl_list_entry_first(&context->gauges, struct cmt_gauge, _head); + TEST_ASSERT(cfl_list_size(&gauge->map->metrics) == 3); + + internal = relay_copy(context, CMT_TRUE); + external = relay_copy(context, CMT_FALSE); + cfl_kvlist_destroy(gauge->map->otlp_metadata); + gauge->map->otlp_metadata = NULL; + cfl_list_foreach(head, &gauge->map->metrics) { + sample = cfl_list_entry(head, struct cmt_metric, _head); + cfl_kvlist_destroy(sample->otlp_metadata); + sample->otlp_metadata = NULL; + } + cfl_kvlist_destroy(context->internal_metadata); + context->internal_metadata = internal; + cfl_kvlist_destroy(context->external_metadata); + context->external_metadata = external; + + if (swap) { + first = gauge->map->metrics.next; + cfl_list_del(first); + cfl_list_add_after(first, gauge->map->metrics.next, &gauge->map->metrics); + } + cfl_list_foreach(head, &gauge->map->metrics) { + if (index++ == drop_index) { + cmt_map_metric_destroy(cfl_list_entry(head, struct cmt_metric, _head)); + break; + } + } + + TEST_ASSERT(cmt_encode_msgpack_create(context, &relay->buffer, &size) == 0); + offset = 0; + TEST_ASSERT(cmt_decode_msgpack_create(&relay->decoded, relay->buffer, size, &offset) == 0); + relay->wire = cmt_encode_opentelemetry_create(relay->decoded); + TEST_ASSERT(relay->wire != NULL); + relay->output = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(relay->wire), (const uint8_t *) relay->wire); + TEST_ASSERT(relay->output != NULL); +} + +static void typed_relay_destroy(struct typed_relay *relay) +{ + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked( + relay->output, NULL); + cmt_encode_opentelemetry_destroy(relay->wire); + cmt_decode_msgpack_destroy(relay->decoded); + cmt_encode_msgpack_destroy(relay->buffer); + cmt_decode_opentelemetry_destroy(&relay->contexts); + cfl_sds_destroy(relay->payload); +} + +/* Each exported point carries the flags and attribute type recorded for its + * value, or, when values cannot tell points apart, neither. */ +static void check_typed_points(struct typed_relay *relay, size_t expected_points, + const double *values, const uint32_t *flags, + int distinguishable) +{ + static const int cases[] = { + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE, + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE, + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE + }; + otlp_metric *metric; + otlp_number_point *point; + size_t index; + size_t source; + + metric = relay->output->resource_metrics[0]->scope_metrics[0]->metrics[0]; + TEST_CHECK(metric->gauge->n_data_points == expected_points); + for (index = 0; index < metric->gauge->n_data_points; index++) { + point = metric->gauge->data_points[index]; + TEST_ASSERT(point->n_attributes == 1); + if (!distinguishable) { + TEST_CHECK_(point->flags == 0, "point %g carries no recorded flags", + point->as_double); + TEST_CHECK_(point->attributes[0]->value->value_case == cases[1], + "point %g keeps its label-derived attribute", point->as_double); + continue; + } + for (source = 0; source < 3 && values[source] != point->as_double; source++); + TEST_ASSERT(source < 3); + TEST_CHECK_(point->flags == flags[source], "point %g keeps its own flags", + point->as_double); + TEST_CHECK_(point->attributes[0]->value->value_case == cases[source], + "point %g keeps its own attribute type", point->as_double); + } +} + +/* Points that differ only by attribute type share labels; their values tell + * them apart through any relay edit. Points identical in value too cannot be + * told apart, so none of them may take metadata recorded for another, from + * the native sidecar or from the legacy tree. */ +static void test_attribute_type_identity_after_relay(void) +{ + static const double distinct[] = {1, 2, 3}; + static const double equal[] = {5, 5, 5}; + static const uint32_t first_flagged[] = {1, 0, 0}; + static const uint32_t last_flagged[] = {0, 0, 1}; + struct typed_relay relay; + + /* drop the first point */ + typed_relay_run(&relay, typed_points(distinct, first_flagged), 0, CMT_FALSE); + check_typed_points(&relay, 2, distinct, first_flagged, CMT_TRUE); + typed_relay_destroy(&relay); + + /* swap the first two points: count and identities are unchanged */ + typed_relay_run(&relay, typed_points(distinct, first_flagged), -1, CMT_TRUE); + check_typed_points(&relay, 3, distinct, first_flagged, CMT_TRUE); + typed_relay_destroy(&relay); + typed_relay_run(&relay, typed_points(equal, first_flagged), -1, CMT_TRUE); + check_typed_points(&relay, 3, equal, first_flagged, CMT_FALSE); + typed_relay_destroy(&relay); + + /* drop the flagged last point: its legacy entry shares the survivors' + * series hash and time */ + typed_relay_run(&relay, typed_points(distinct, last_flagged), 2, CMT_FALSE); + check_typed_points(&relay, 2, distinct, last_flagged, CMT_TRUE); + typed_relay_destroy(&relay); + typed_relay_run(&relay, typed_points(equal, last_flagged), 2, CMT_FALSE); + check_typed_points(&relay, 2, equal, last_flagged, CMT_FALSE); + typed_relay_destroy(&relay); + + test_alloc_release(); +} + +/* SDKs do not order attributes; series with the same keys in a different + * order must keep their own values. */ +static void test_attribute_order(void) +{ + run_case(single_metric(metric_gauge("order", "attribute order", "1", 2, + number_double(0, 100, 1, 2, kv_list(2, kv("a", any_string("1")), kv("b", any_string("2")))), + number_double(0, 100, 2, 2, kv_list(2, kv("b", any_string("3")), kv("a", any_string("4"))))))); +} + +/* Points of one metric with different attribute key sets. */ +static void test_attribute_key_sets(void) +{ + run_case(single_metric(metric_gauge("key_sets", "key sets", "1", 3, + number_double(0, 100, 1, 0, NULL), + number_double(0, 100, 2, 1, attrs1("a", any_string("x"))), + number_double(0, 100, 3, 2, kv_list(2, kv("a", any_string("x")), + kv("b", any_string("y"))))))); +} + +/* A collector forwarding several services in one request. */ +static void test_multiple_resources_and_scopes(void) +{ + run_case(request(2, + resource(1, attrs1("service.name", any_string("cart")), 2, + scope("lib.http", "1.0", 0, NULL, 1, + metric_gauge("shared.name", "from http", "1", 1, + number_double(0, 100, 1, 0, NULL))), + scope("lib.db", "2.0", 1, attrs1("db.system", any_string("postgresql")), 1, + metric_sum("shared.name", "from db", "1", TEMPORALITY_CUMULATIVE, 1, 1, + number_int(0, 100, 5, 0, NULL)))), + resource(1, attrs1("service.name", any_string("payment")), 1, + scope("lib.http", "1.0", 0, NULL, 1, + metric_gauge("shared.name", "from http", "1", 1, + number_double(0, 100, 2, 0, NULL)))))); +} + +static otlp_request *details_request(const char *description) +{ + otlp_request *input; + + input = single_metric(metric_gauge("details", description, "", 1, + number_double(0, 100, 1, 0, NULL))); + input->resource_metrics[0]->schema_url = "https://opentelemetry.io/schemas/1.24.0"; + input->resource_metrics[0]->resource->dropped_attributes_count = 3; + input->resource_metrics[0]->scope_metrics[0]->schema_url = + "https://opentelemetry.io/schemas/1.21.0"; + input->resource_metrics[0]->scope_metrics[0]->scope->dropped_attributes_count = 1; + input->resource_metrics[0]->scope_metrics[0]->scope->n_attributes = 1; + input->resource_metrics[0]->scope_metrics[0]->scope->attributes = + attrs1("scope.kind", any_string("auto")); + return input; +} + +/* Schema URLs, dropped counts and scope attributes. CMetrics families require + * a description, so an empty one is emitted as "-" (pinned). */ +static void test_resource_and_scope_details(void) +{ + check_pipelines(details_request(""), details_request("-"), ALL_PIPELINES); + test_alloc_release(); +} + +static void test_metric_metadata(void) +{ + otlp_metric *metric; + + metric = metric_gauge("with.metadata", "has metadata", "By", 1, + number_double(0, 100, 1, 0, NULL)); + metric->n_metadata = 2; + metric->metadata = kv_list(2, kv("prometheus.type", any_string("gauge")), + kv("weight", any_int(3))); + run_case(single_metric(metric)); +} + +static void test_number_exemplars(void) +{ + otlp_number_point *point; + + point = number_int(1000, 2000, 10, 1, attrs1("route", any_string("/"))); + point->n_exemplars = 2; + point->exemplars = exemplar_list(2, + exemplar_int(1500, 4), + exemplar_double(1600, 6.5, "fedcba9876543210", "76543210", 2, + kv_list(2, kv("user", any_string("u2")), kv("n", any_int(1))))); + run_case(single_metric(metric_sum("exemplars", "with exemplars", "1", + TEMPORALITY_CUMULATIVE, 1, 1, point))); +} + +/* Large cardinality as produced by busy collectors. */ +static void test_many_series(void) +{ + otlp_metric *metric; + char *values; + size_t index; + + metric = metric_gauge("many.series", "cardinality", "1", 0); + metric->gauge->n_data_points = 2000; + metric->gauge->data_points = test_alloc(sizeof(otlp_number_point *) * 2000); + values = test_alloc(2000 * 8); + for (index = 0; index < 2000; index++) { + snprintf(&values[index * 8], 8, "%zu", index); + metric->gauge->data_points[index] = + number_double(0, 100, (double) index, 1, + attrs1("series", any_string(&values[index * 8]))); + } + run_case(single_metric(metric)); +} + +/* Every prefix and single byte corruption of a real request must be handled + * without crashing; whatever decodes must also survive a MessagePack hop. */ +static void test_truncated_and_corrupted_requests(void) +{ + static const double bounds[] = {1, 2}; + static const uint64_t counts[] = {1, 2, 3}; + struct cfl_list contexts; + struct cfl_list *head; + struct cmt *context; + struct cmt *decoded; + cfl_sds_t payload; + cfl_sds_t copy; + size_t length; + size_t offset; + int result; + + payload = pack_request(request(1, resource(RESOURCE_ATTRIBUTES, 1, + scope("s", "1", 1, attrs1("k", any_string("v")), 3, + metric_sum("c", "d", "1", TEMPORALITY_CUMULATIVE, 1, 1, + number_int(1, 2, 3, 1, attrs1("a", any_int(1)))), + metric_histogram("h", "s", TEMPORALITY_CUMULATIVE, 1, + histogram_point(1, 2, 2, bounds, counts, 1, attrs1("a", any_bool(1)))), + metric_exp_histogram("e", TEMPORALITY_DELTA, 1, + exp_point(1, 2, 1, 1, 0, exp_buckets(-1, 3, counts), NULL, 0, NULL)))))); + + for (length = 0; length <= cfl_sds_len(payload) * 2; length++) { + copy = cfl_sds_create_len(payload, length <= cfl_sds_len(payload) ? + length : cfl_sds_len(payload)); + if (length > cfl_sds_len(payload)) { + copy[length - cfl_sds_len(payload) - 1] ^= 0x5a; + } + offset = 0; + result = cmt_decode_opentelemetry_create(&contexts, copy, cfl_sds_len(copy), &offset); + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cfl_list_foreach(head, &contexts) { + context = cfl_list_entry(head, struct cmt, _head); + decoded = msgpack_hop(context, CMT_FALSE); + if (decoded != NULL) { + cmt_decode_msgpack_destroy(decoded); + } + } + cmt_decode_opentelemetry_destroy(&contexts); + } + cfl_sds_destroy(copy); + } + cfl_sds_destroy(payload); + test_alloc_release(); +} + +/* Containers without data must not produce data points or fail. */ +static void test_empty_containers(void) +{ + otlp_request *input; + + input = request(3, + resource(RESOURCE_ATTRIBUTES, 0), + resource(RESOURCE_ATTRIBUTES, 1, scope("empty", "1", 0, NULL, 0)), + resource(RESOURCE_ATTRIBUTES, 1, scope("s", "1", 0, NULL, 1, + metric_gauge("no.points", "", "", 0)))); + check_pipelines(input, request(0), ALL_PIPELINES); + test_alloc_release(); +} + +TEST_LIST = { + {"sum_monotonic_cumulative", test_sum_monotonic_cumulative}, + {"sum_non_monotonic", test_sum_non_monotonic}, + {"sum_delta", test_sum_delta}, + {"gauge_integer_extremes", test_gauge_integer_extremes}, + {"gauge_special_doubles", test_gauge_special_doubles}, + {"point_flags_and_timestamps", test_point_flags_and_timestamps}, + {"histogram_explicit_bounds", test_histogram_explicit_bounds}, + {"histogram_delta_without_sum", test_histogram_delta_without_sum}, + {"histogram_without_bounds_is_rejected", test_histogram_without_bounds_is_rejected}, + {"exponential_histogram_layouts", test_exponential_histogram_layouts}, + {"summary_quantiles", test_summary_quantiles}, + {"summary_without_quantiles", test_summary_without_quantiles}, + {"typed_attributes", test_typed_attributes}, + {"attribute_type_identity", test_attribute_type_identity}, + {"attribute_type_identity_after_relay", test_attribute_type_identity_after_relay}, + {"distribution_type_identity", test_distribution_type_identity}, + {"attribute_order", test_attribute_order}, + {"attribute_key_sets", test_attribute_key_sets}, + {"multiple_resources_and_scopes", test_multiple_resources_and_scopes}, + {"resource_and_scope_details", test_resource_and_scope_details}, + {"metric_metadata", test_metric_metadata}, + {"number_exemplars", test_number_exemplars}, + {"many_series", test_many_series}, + {"truncated_and_corrupted_requests", test_truncated_and_corrupted_requests}, + {"empty_containers", test_empty_containers}, + {0} +};