diff --git a/benchmarks/compare_fluent_bit_otelcol.py b/benchmarks/compare_fluent_bit_otelcol.py new file mode 100644 index 00000000000..c79e5df1e5e --- /dev/null +++ b/benchmarks/compare_fluent_bit_otelcol.py @@ -0,0 +1,745 @@ +#!/usr/bin/env python3 +"""Compare Fluent Bit and Collector OTLP/HTTP log ingestion to null sinks. + +Measurement model: + +- The load generator runs as one process per client so it is not limited by + the Python GIL. Its own CPU use is reported so a generator ceiling is + visible in the results. +- Server and generator CPU sets must not share a physical core; the harness + checks the SMT sibling map and refuses to run otherwise. It also refuses to + start while other processes keep those CPUs busy. +- Fluent Bit refreshes its metrics endpoints once per second, the Collector + serves live counters. To measure both the same way, throughput is the + slope of the completed-records counter over a steady-state window + (by default 10% to 90% of the measured records), using the time at which + each new counter value was first observed. End-to-end completion time is + still reported, but it is quantized to one second for Fluent Bit. +""" + +import argparse +import csv +import hashlib +import http.client +import json +import math +import multiprocessing +import os +from pathlib import Path +import re +import socket +import statistics +import subprocess +import tempfile +import threading +import time +import urllib.request + + +ENCODINGS = ("json", "protobuf") +PIPELINES = ("ingest", "modified") +SAMPLE_INTERVAL = 0.05 +CLIENT_DEADLINE = 900 +CLIENT_MAX_CONSECUTIVE_FAILURES = 200 + + +def free_port(): + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def parse_cpus(text): + cpus = set() + for item in text.split(","): + if "-" in item: + low, high = item.split("-") + cpus.update(range(int(low), int(high) + 1)) + else: + cpus.add(int(item)) + return cpus + + +def physical_core(cpu): + path = Path(f"/sys/devices/system/cpu/cpu{cpu}/topology/thread_siblings_list") + return min(parse_cpus(path.read_text().strip())) + + +def check_topology(server_cpus, generator_cpus): + server_cores = {physical_core(cpu) for cpu in server_cpus} + generator_cores = {physical_core(cpu) for cpu in generator_cpus} + shared = server_cores & generator_cores + if shared: + raise SystemExit(f"server and generator CPUs share physical cores {sorted(shared)}") + + +def cpu_busy_fraction(cpus, interval=1.0): + def read(): + values = {} + for line in Path("/proc/stat").read_text().splitlines(): + if line.startswith("cpu") and line[3:4].isdigit(): + fields = line.split() + numbers = [int(value) for value in fields[1:]] + values[int(fields[0][3:])] = (numbers[3] + numbers[4], sum(numbers)) + return values + + before = read() + time.sleep(interval) + after = read() + idle = sum(after[cpu][0] - before[cpu][0] for cpu in cpus) + total = sum(after[cpu][1] - before[cpu][1] for cpu in cpus) + return 1.0 - idle / total if total else 0.0 + + +def wait_quiet(cpus, limit=0.10, attempts=30): + for _ in range(attempts): + busy = cpu_busy_fraction(cpus) + if busy <= limit: + return busy + raise RuntimeError(f"benchmark CPUs stayed busy ({busy:.0%}); stop other workloads first") + + +def percentile(values, percentile_value): + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, math.ceil(percentile_value * len(ordered)) - 1)) + return ordered[index] + + +def process_cpu_seconds(pid): + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + ticks = os.sysconf("SC_CLK_TCK") + return (int(fields[11]) + int(fields[12])) / ticks + + +def thread_cpu_ticks(pid): + threads = {} + for task in Path(f"/proc/{pid}/task").iterdir(): + try: + fields = (task / "stat").read_text().rsplit(")", 1)[1].split() + name = (task / "comm").read_text().strip() + except (FileNotFoundError, ProcessLookupError): + continue + threads[int(task.name)] = (name, int(fields[11]) + int(fields[12])) + return threads + + +def process_rss_bytes(pid): + for line in Path(f"/proc/{pid}/status").read_text().splitlines(): + if line.startswith("VmRSS:"): + return int(line.split()[1]) * 1024 + raise RuntimeError("VmRSS was absent") + + +def scrape(url): + with urllib.request.urlopen(url, timeout=2) as response: + return response.read().decode() + + +def prometheus_value(text, metric): + pattern = re.compile(rf"^{re.escape(metric)}(?:\{{[^}}]*\}})?\s+([0-9.eE+-]+)$", re.MULTILINE) + values = [float(match.group(1)) for match in pattern.finditer(text)] + return sum(values) + + +def fluent_metrics(port): + data = json.loads(scrape(f"http://127.0.0.1:{port}/api/v1/metrics")) + output = data.get("output", {}).get("null.0", {}) + input_metrics = data.get("input", {}).get("opentelemetry.0", {}) + return { + "accepted": int(input_metrics.get("records", 0)), + "completed": int(output.get("proc_records", 0)), + "dropped": int(output.get("dropped_records", 0)), + "errors": int(output.get("errors", 0)), + "retries": int(output.get("retries", 0)), + "queue": 0, + } + + +def collector_metrics(port): + text = scrape(f"http://127.0.0.1:{port}/metrics") + return { + "accepted": int(prometheus_value(text, "otelcol_receiver_accepted_log_records")), + "completed": int(prometheus_value(text, "otelcol_null_discarded_log_records")), + "dropped": int(prometheus_value(text, "otelcol_exporter_enqueue_failed_log_records")), + "errors": int(prometheus_value(text, "otelcol_receiver_refused_log_records")), + "retries": int(prometheus_value(text, "otelcol_exporter_send_failed_log_records")), + "queue": int(prometheus_value(text, "otelcol_exporter_queue_size")), + } + + +def wait_ready(metrics_function, port, process): + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("server exited during startup") + try: + metrics_function(port) + return + except Exception: + time.sleep(0.05) + raise TimeoutError("metrics endpoint did not become ready") + + +def wait_completed(metrics_function, port, process, expected, timeout): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError("server exited before drain") + try: + current = metrics_function(port) + except Exception: + # Fluent Bit answers 404 while no metrics snapshot is published + time.sleep(0.01) + continue + if current["completed"] >= expected and current["queue"] == 0: + if current["completed"] != expected: + raise RuntimeError(f"completion exceeded expected count: {current}") + return current + time.sleep(0.01) + raise TimeoutError(f"completion did not reach {expected}") + + +def make_payloads(batch_records): + attributes = [ + {"key": f"field{index}", "value": {"stringValue": "representative attribute"}} + for index in range(10) + ] + log_record = { + "timeUnixNano": "1700000000000000000", + "severityNumber": 9, + "severityText": "INFO", + "body": {"stringValue": "representative benchmark log message"}, + "attributes": attributes, + } + request = { + "resourceLogs": [{ + "resource": {"attributes": [ + {"key": "service.name", "value": {"stringValue": "benchmark"}} + ]}, + "scopeLogs": [{ + "scope": {"name": "benchmark", "version": "1.0"}, + "logRecords": [log_record] * batch_records, + }], + }] + } + json_payload = json.dumps(request, separators=(",", ":")).encode() + from google.protobuf.json_format import ParseDict + from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + protobuf_payload = ParseDict(request, ExportLogsServiceRequest()).SerializeToString() + return {"json": json_payload, "protobuf": protobuf_payload} + + +def system_kind(name): + if "otelcol" in name: + return "collector" + return "fluent-bit" + + +def fluent_config(input_port, metrics_port, modified, flush_seconds, storage_dir): + processor = "" + service_storage = "" + input_storage = "" + if storage_dir: + service_storage = f""" storage.path: {storage_dir} + storage.sync: normal + storage.checksum: off +""" + input_storage = " storage.type: filesystem\n" + if modified: + processor = """ processors: + logs: + - name: content_modifier + context: otel_log_attributes + action: upsert + key: bench.added + value: yes +""" + return f"""service: + flush: {flush_seconds} + grace: 30 + log_level: error + http_server: on + http_listen: 127.0.0.1 + http_port: {metrics_port} +{service_storage}pipeline: + inputs: + - name: opentelemetry + tag: benchmark + listen: 127.0.0.1 + port: {input_port} + http2: off + tls: off + threaded: true + http_server.workers: 4 +{input_storage}{processor} outputs: + - name: null + match: '*' +""" + + +def collector_config(input_port, metrics_port, modified, flush_seconds, storage_dir): + processor_definition = "" + extension_definition = "" + queue_storage = "" + service_extensions = "" + if storage_dir: + extension_definition = f"""extensions: + file_storage: + directory: {storage_dir} + +""" + queue_storage = " storage: file_storage\n" + service_extensions = " extensions: [file_storage]\n" + pipeline_processor = "" + if modified: + processor_definition = """processors: + transform/add_attribute: + error_mode: propagate + log_statements: + - context: log + statements: + - set(attributes[\"bench.added\"], \"yes\") + +""" + pipeline_processor = " processors: [transform/add_attribute]\n" + return f"""receivers: + otlp: + protocols: + http: + endpoint: 127.0.0.1:{input_port} + +{extension_definition}{processor_definition}exporters: + null/fluent_bit: + sending_queue: + enabled: true + num_consumers: 1 + queue_size: 1000000 + sizer: items +{queue_storage} batch: + enabled: true + flush_timeout: {flush_seconds}s + min_size: 8192 + sizer: items + +service: +{service_extensions} telemetry: + logs: + level: error + metrics: + level: detailed + readers: + - pull: + exporter: + prometheus: + host: 127.0.0.1 + port: {metrics_port} + pipelines: + logs: + receivers: [otlp] +{pipeline_processor} exporters: [null/fluent_bit] +""" + + +def send_requests(task): + """Generator process body: one keep-alive connection, bounded retries.""" + port, payload, requests, content_type = task + latencies = [] + retries = 0 + failures = 0 + cpu_start = time.process_time() + deadline = time.monotonic() + CLIENT_DEADLINE + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30) + try: + for _ in range(requests): + started = time.monotonic_ns() + while True: + if time.monotonic() > deadline: + raise TimeoutError("client deadline exceeded") + try: + connection.request("POST", "/v1/logs", payload, + {"Content-Type": content_type}) + response = connection.getresponse() + body = response.read() + if response.status in (200, 201): + failures = 0 + break + if response.status not in (429, 503): + raise RuntimeError(f"HTTP {response.status}: {body!r}") + except (ConnectionError, TimeoutError, http.client.HTTPException, OSError): + connection.close() + failures += 1 + if failures >= CLIENT_MAX_CONSECUTIVE_FAILURES: + raise + connection = http.client.HTTPConnection("127.0.0.1", port, timeout=30) + retries += 1 + time.sleep(0.0005) + latencies.append((time.monotonic_ns() - started) / 1000) + finally: + connection.close() + return latencies, retries, time.process_time() - cpu_start + + +def run_clients(pool, clients, port, payload, requests_per_client, content_type): + tasks = [(port, payload, requests_per_client, content_type)] * clients + return pool.map(send_requests, tasks, chunksize=1) + + +def sample_process(pid, done, samples, metrics_function, metrics_port): + def take(): + samples.append((time.monotonic(), process_cpu_seconds(pid), + process_rss_bytes(pid), metrics_function(metrics_port), + thread_cpu_ticks(pid))) + + while not done.is_set(): + try: + take() + except FileNotFoundError: + return + except Exception: + pass + done.wait(SAMPLE_INTERVAL) + try: + take() + except Exception: + pass + + +def peak_cpu_cores(samples): + peak = 0.0 + for start_index, start in enumerate(samples): + for end in samples[start_index + 1:]: + elapsed = end[0] - start[0] + if elapsed >= 1.0: + peak = max(peak, (end[1] - start[1]) / elapsed) + break + return peak + + +def change_points(samples, key): + """Samples where the counter first shows a new value.""" + points = [] + previous = None + for sample in samples: + value = sample[3][key] + if value != previous: + points.append(sample) + previous = value + return points + + +def steady_window(samples, key, base, records, low, high): + points = change_points(samples, key) + lower = base + records * low + upper = base + records * high + start = next((point for point in points if point[3][key] >= lower), None) + end = next((point for point in reversed(points) if point[3][key] <= upper), None) + if start is None or end is None or end[0] - start[0] < 2.0: + raise RuntimeError(f"steady-state window for {key} is too short; raise --records") + return start, end + + +def thread_profile(start, end, top=4): + ticks = os.sysconf("SC_CLK_TCK") + elapsed = end[0] - start[0] + usage = {} + for tid, (name, value) in end[4].items(): + before = start[4].get(tid, (name, 0))[1] + usage[(tid, name)] = (value - before) / ticks / elapsed + ordered = sorted(usage.items(), key=lambda item: item[1], reverse=True) + busiest = ordered[0][1] if ordered else 0.0 + text = "|".join(f"{name}:{value:.2f}" for (tid, name), value in ordered[:top]) + return busiest, text + + +def run_once(system, binary, encoding, pipeline, repetition, args, payload, pool): + records = args.records + clients = args.clients + batch_records = args.batch_records + warmup_records = args.warmup_records + input_port = free_port() + metrics_port = free_port() + modified = pipeline == "modified" + kind = system_kind(system) + is_fluent_bit = kind != "collector" + metrics_function = fluent_metrics if is_fluent_bit else collector_metrics + content_type = "application/json" if encoding == "json" else "application/x-protobuf" + quiet_busy = wait_quiet(args.server_cpu_set | args.generator_cpu_set) + with tempfile.TemporaryDirectory(prefix="otel-compare-", dir=args.work_dir) as directory: + storage_dir = "" + if args.storage == "filesystem": + storage_dir = str(Path(directory) / "storage") + os.mkdir(storage_dir) + if is_fluent_bit: + config_text = fluent_config(input_port, metrics_port, modified, + args.flush_seconds, storage_dir) + else: + config_text = collector_config(input_port, metrics_port, modified, + args.flush_seconds, storage_dir) + config_path = Path(directory) / "config.yaml" + log_path = Path(directory) / "server.log" + config_path.write_text(config_text) + environment = os.environ.copy() + if not is_fluent_bit: + environment["GOMAXPROCS"] = str(len(args.server_cpu_set)) + with log_path.open("w+") as log: + process = subprocess.Popen( + ["taskset", "-c", args.server_cpus, str(binary), "--config", str(config_path)], + stdout=log, stderr=log, env=environment, + ) + try: + wait_ready(metrics_function, metrics_port, process) + run_clients(pool, clients, input_port, payload, + warmup_records // batch_records // clients, content_type) + wait_completed(metrics_function, metrics_port, process, warmup_records, 120) + idle_rss = process_rss_bytes(process.pid) + start_cpu = process_cpu_seconds(process.pid) + start = time.monotonic() + done = threading.Event() + samples = [] + monitor = threading.Thread(target=sample_process, + args=(process.pid, done, samples, + metrics_function, metrics_port), daemon=True) + monitor.start() + client_results = run_clients(pool, clients, input_port, payload, + records // batch_records // clients, + content_type) + send_end = time.monotonic() + final_metrics = wait_completed(metrics_function, metrics_port, process, + warmup_records + records, 600) + end = time.monotonic() + end_cpu = process_cpu_seconds(process.pid) + done.set() + monitor.join() + + accepted_delta = final_metrics["accepted"] - warmup_records + completed_delta = final_metrics["completed"] - warmup_records + if accepted_delta != records or completed_delta != records: + raise RuntimeError(f"accounting mismatch: {final_metrics}") + # A full Collector queue refuses requests, the clients resend them + # like Fluent Bit 429/503 answers; the accounting above proves no loss + if is_fluent_bit and (final_metrics["dropped"] or final_metrics["errors"]): + raise RuntimeError(f"loss/refusal counters are nonzero: {final_metrics}") + + latencies = [latency for result in client_results for latency in result[0]] + retries = sum(result[1] for result in client_results) + generator_cpu = sum(result[2] for result in client_results) + samples = [sample for sample in samples if sample[3] is not None] + rss_values = [sample[2] for sample in samples] + peak_rss_sample = max(samples, key=lambda sample: sample[2]) + peak_backlog = max(sample[3]["accepted"] - sample[3]["completed"] + for sample in samples) + + completed_start, completed_end = steady_window( + samples, "completed", warmup_records, records, + args.window_low, args.window_high) + steady_seconds = completed_end[0] - completed_start[0] + steady_records = completed_end[3]["completed"] - completed_start[3]["completed"] + steady_cpu = completed_end[1] - completed_start[1] + busiest_thread, threads_text = thread_profile(completed_start, completed_end) + + accepted_start, accepted_end = steady_window( + samples, "accepted", warmup_records, records, + args.window_low, args.window_high) + steady_accepted_rate = ( + (accepted_end[3]["accepted"] - accepted_start[3]["accepted"]) / + (accepted_end[0] - accepted_start[0])) + + elapsed = end - start + send_elapsed = send_end - start + cpu = end_cpu - start_cpu + return { + "system": system, + "repetition": repetition, + "pipeline": pipeline, + "encoding": encoding, + "workers_or_gomaxprocs": len(args.server_cpu_set), + "clients": clients, + "batch_records": batch_records, + "payload_bytes": len(payload), + "payload_sha256": hashlib.sha256(payload).hexdigest(), + "offered_records": records, + "accepted_records": accepted_delta, + "completed_records": completed_delta, + "client_retries": retries, + "dropped_records": final_metrics["dropped"], + "refused_records": final_metrics["errors"], + "pre_run_cpu_busy": quiet_busy, + "send_seconds": send_elapsed, + "completion_seconds": elapsed, + "steady_window_seconds": steady_seconds, + "steady_completed_records_per_second": steady_records / steady_seconds, + "steady_accepted_records_per_second": steady_accepted_rate, + "steady_cpu_cores": steady_cpu / steady_seconds, + "steady_records_per_cpu_second": steady_records / steady_cpu, + "busiest_thread_cpu": busiest_thread, + "top_threads": threads_text, + "end_to_end_records_per_second": records / elapsed, + "payload_mib_per_second": (steady_records / batch_records * len(payload) / + steady_seconds / 1048576), + "cpu_seconds": cpu, + "cpu_seconds_per_million_records": cpu / records * 1_000_000, + "average_cpu_cores": cpu / elapsed, + "peak_cpu_cores_1s": peak_cpu_cores(samples), + "records_per_cpu_second": records / cpu, + "generator_cpu_seconds": generator_cpu, + "generator_cpu_utilization": (generator_cpu / send_elapsed / + len(args.generator_cpu_set)), + "idle_rss_mib": idle_rss / 1048576, + "steady_rss_mib": statistics.median(rss_values) / 1048576, + "peak_rss_mib": max(rss_values) / 1048576, + "peak_processing_backlog_records": peak_backlog, + "backlog_at_peak_rss_records": ( + peak_rss_sample[3]["accepted"] - + peak_rss_sample[3]["completed"]), + "latency_p50_us": percentile(latencies, 0.50), + "latency_p95_us": percentile(latencies, 0.95), + "latency_p99_us": percentile(latencies, 0.99), + "final_queue_items": final_metrics["queue"], + } + except Exception: + log.flush() + log.seek(0) + print(log.read(), flush=True) + raise + finally: + process.terminate() + try: + process.wait(timeout=15) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +SUMMARY_FIELDS = ( + "steady_completed_records_per_second", "steady_accepted_records_per_second", + "steady_cpu_cores", "steady_records_per_cpu_second", "busiest_thread_cpu", + "end_to_end_records_per_second", "payload_mib_per_second", + "cpu_seconds_per_million_records", "records_per_cpu_second", "average_cpu_cores", + "peak_cpu_cores_1s", "generator_cpu_utilization", + "idle_rss_mib", "steady_rss_mib", "peak_rss_mib", "peak_processing_backlog_records", + "latency_p50_us", "latency_p95_us", "latency_p99_us", +) + + +def summarize(rows, systems): + summary = [] + for pipeline in PIPELINES: + for encoding in ENCODINGS: + for system in systems: + selected = [row for row in rows if row["pipeline"] == pipeline + and row["encoding"] == encoding and row["system"] == system] + if not selected: + continue + output = { + "pipeline": pipeline, + "encoding": encoding, + "system": system, + "repetitions": len(selected), + } + for field in SUMMARY_FIELDS: + values = [float(row[field]) for row in selected] + output[f"median_{field}"] = statistics.median(values) + output[f"min_{field}"] = min(values) + output[f"max_{field}"] = max(values) + output["total_retries"] = sum(int(row["client_retries"]) for row in selected) + output["total_dropped"] = sum(int(row["dropped_records"]) for row in selected) + output["total_refused"] = sum(int(row["refused_records"]) for row in selected) + summary.append(output) + return summary + + +def write_csv(path, rows): + with path.open("w", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + + +def output_path(prefix, suffix): + return Path(f"{prefix}{suffix}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--system", action="append", required=True, metavar="NAME=PATH", + help="binary to measure; names containing 'otelcol' run as the " + "Collector, any other name as Fluent Bit") + parser.add_argument("--output-prefix", type=Path, required=True) + parser.add_argument("--records", type=int, default=30_000_000) + parser.add_argument("--warmup-records", type=int, default=320_000) + parser.add_argument("--batch-records", type=int, default=100) + parser.add_argument("--clients", type=int, default=16) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--flush-seconds", type=float, default=0.5) + parser.add_argument("--server-cpus", default="0-3") + parser.add_argument("--generator-cpus", default="4-7,12-15") + parser.add_argument("--window-low", type=float, default=0.10) + parser.add_argument("--window-high", type=float, default=0.90) + parser.add_argument("--pipelines", default=",".join(PIPELINES)) + parser.add_argument("--encodings", default=",".join(ENCODINGS)) + parser.add_argument("--storage", choices=("memory", "filesystem"), default="memory", + help="buffer records in memory, or in filesystem chunks (Fluent Bit) " + "and a persistent sending queue (Collector file_storage)") + parser.add_argument("--work-dir", type=Path, default=None, + help="directory for configurations and filesystem storage " + "(default: the system temporary directory)") + args = parser.parse_args() + divisor = args.batch_records * args.clients + if args.records % divisor or args.warmup_records % divisor: + parser.error("record and warm-up counts must divide evenly across batches and clients") + + args.server_cpu_set = parse_cpus(args.server_cpus) + args.generator_cpu_set = parse_cpus(args.generator_cpus) + check_topology(args.server_cpu_set, args.generator_cpu_set) + os.sched_setaffinity(0, args.generator_cpu_set) + + binaries = {} + for item in args.system: + name, path = item.split("=", 1) + binaries[name] = Path(path).resolve() + systems = tuple(binaries) + pipelines = tuple(args.pipelines.split(",")) + encodings = tuple(args.encodings.split(",")) + + payloads = make_payloads(args.batch_records) + manifest = { + "created_unix": time.time(), + "server_cpus": sorted(args.server_cpu_set), + "generator_cpus": sorted(args.generator_cpu_set), + "arguments": {key: str(value) for key, value in vars(args).items()}, + "binaries": {system: str(path) for system, path in binaries.items()}, + "binary_sha256": { + system: hashlib.sha256(path.read_bytes()).hexdigest() + for system, path in binaries.items() + }, + "payloads": { + encoding: {"bytes": len(payload), + "sha256": hashlib.sha256(payload).hexdigest()} + for encoding, payload in payloads.items() + }, + "transformation": "upsert log attribute bench.added as string yes", + } + args.output_prefix.parent.mkdir(parents=True, exist_ok=True) + output_path(args.output_prefix, ".manifest.json").write_text( + json.dumps(manifest, indent=2, default=str) + "\n") + + rows = [] + context = multiprocessing.get_context("fork") + with context.Pool(args.clients) as pool: + for repetition in range(1, args.runs + 1): + for pipeline in pipelines: + for encoding in encodings: + offset = (repetition - 1) % len(systems) + order = systems[offset:] + systems[:offset] + for system in order: + row = run_once(system, binaries[system], encoding, pipeline, + repetition, args, payloads[encoding], pool) + rows.append(row) + write_csv(output_path(args.output_prefix, ".raw.csv"), rows) + print(json.dumps(row, sort_keys=True), flush=True) + write_csv(output_path(args.output_prefix, ".summary.csv"), summarize(rows, systems)) + + +if __name__ == "__main__": + main() diff --git a/include/fluent-bit/flb_downstream.h b/include/fluent-bit/flb_downstream.h index 6bb446c056d..12c1a0b3542 100644 --- a/include/fluent-bit/flb_downstream.h +++ b/include/fluent-bit/flb_downstream.h @@ -98,6 +98,18 @@ int flb_downstream_conn_event_accept( void *accept_callback_data, flb_connection_event_callback event_callback, int mask); + +/* + * Same as flb_downstream_conn_event_accept() for a socket that was already + * accepted, for example by another worker. The socket is always consumed. + */ +int flb_downstream_conn_event_adopt( + struct flb_downstream *stream, + flb_sockfd_t fd, + flb_connection_accept_callback accept_callback, + void *accept_callback_data, + flb_connection_event_callback event_callback, + int mask); int flb_downstream_conn_event_register(struct flb_connection *connection, int (*callback)(void *data), int mask); diff --git a/include/fluent-bit/flb_input.h b/include/fluent-bit/flb_input.h index 4824ea824af..b0086eff626 100644 --- a/include/fluent-bit/flb_input.h +++ b/include/fluent-bit/flb_input.h @@ -957,6 +957,12 @@ int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, const char *tag, size_t tag_len, void *buf, size_t buf_size, size_t allocation_size); +int flb_input_ingress_queue_log_processed_take_records( + struct flb_input_instance *ins, + size_t records, + const char *tag, size_t tag_len, + void *buf, size_t buf_size, + size_t allocation_size); int flb_input_ingress_queue_metrics(struct flb_input_instance *ins, const char *tag, size_t tag_len, struct cmt *cmt, size_t payload_size); diff --git a/include/fluent-bit/flb_input_log.h b/include/fluent-bit/flb_input_log.h index fa88551bc7b..8adfb999aec 100644 --- a/include/fluent-bit/flb_input_log.h +++ b/include/fluent-bit/flb_input_log.h @@ -45,4 +45,18 @@ int flb_input_log_append_processed(struct flb_input_instance *ins, size_t tag_len, const void *buf, size_t buf_size); +int flb_input_log_append_processed_routed(struct flb_input_instance *ins, + size_t records, + const char *tag, + size_t tag_len, + const void *buf, + size_t buf_size); + +struct flb_mp_chunk_cobj; +int flb_input_log_chunk_supported(struct flb_input_instance *ins, + int worker_thread); +int flb_input_log_append_chunk(struct flb_input_instance *ins, + const char *tag, size_t tag_len, + struct flb_mp_chunk_cobj *chunk, + int worker_thread); #endif diff --git a/include/fluent-bit/flb_json_tree.h b/include/fluent-bit/flb_json_tree.h new file mode 100644 index 00000000000..7db569621f9 --- /dev/null +++ b/include/fluent-bit/flb_json_tree.h @@ -0,0 +1,441 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_JSON_TREE_H +#define FLB_JSON_TREE_H + +#include +#include + +#include +#include +#include + +#ifdef FLB_HAVE_YYJSON +#include +#endif + +/* + * Read-only JSON tree for traversal in hot paths. A tree is parsed once and its + * strings are borrowed from it, no value is copied or converted. Invalid + * Unicode is replaced like flb_pack_json() does. flb_json_tree_read() returns + * NULL when the input is not a single JSON value or when no tree backend is + * built (FLB_YYJSON=Off), callers must keep another way to read the input. + * + * The accessors are inline so traversal costs the same as using the backend + * directly. They accept NULL and nodes of any type: type checks are false, + * sizes and numbers are zero and strings are NULL. + */ +struct flb_json_tree; +struct flb_json_node; + +enum flb_json_node_type { + FLB_JSON_NODE_NONE = 0, + FLB_JSON_NODE_NULL, + FLB_JSON_NODE_BOOL, + FLB_JSON_NODE_UINT, + FLB_JSON_NODE_SINT, + FLB_JSON_NODE_REAL, + FLB_JSON_NODE_STR, + FLB_JSON_NODE_ARR, + FLB_JSON_NODE_OBJ +}; + +struct flb_json_tree *flb_json_tree_read(const char *input, size_t length); +void flb_json_tree_destroy(struct flb_json_tree *tree); + +#ifdef FLB_HAVE_YYJSON + +static inline yyjson_val *flb_json_node_val(struct flb_json_node *node) +{ + return (yyjson_val *) node; +} + +static inline struct flb_json_node *flb_json_node_wrap(yyjson_val *value) +{ + return (struct flb_json_node *) value; +} + +static inline struct flb_json_node *flb_json_tree_root(struct flb_json_tree *tree) +{ + return flb_json_node_wrap(yyjson_doc_get_root((yyjson_doc *) tree)); +} + +/* one of enum flb_json_node_type, FLB_JSON_NODE_NONE for NULL */ +static inline int flb_json_node_type(struct flb_json_node *node) +{ + yyjson_val *value; + + value = flb_json_node_val(node); + switch (yyjson_get_type(value)) { + case YYJSON_TYPE_NULL: + return FLB_JSON_NODE_NULL; + case YYJSON_TYPE_BOOL: + return FLB_JSON_NODE_BOOL; + case YYJSON_TYPE_NUM: + switch (yyjson_get_subtype(value)) { + case YYJSON_SUBTYPE_UINT: + return FLB_JSON_NODE_UINT; + case YYJSON_SUBTYPE_SINT: + return FLB_JSON_NODE_SINT; + default: + return FLB_JSON_NODE_REAL; + } + case YYJSON_TYPE_STR: + return FLB_JSON_NODE_STR; + case YYJSON_TYPE_ARR: + return FLB_JSON_NODE_ARR; + case YYJSON_TYPE_OBJ: + return FLB_JSON_NODE_OBJ; + default: + return FLB_JSON_NODE_NONE; + } +} + +static inline int flb_json_node_is_null(struct flb_json_node *node) +{ + return yyjson_is_null(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_bool(struct flb_json_node *node) +{ + return yyjson_is_bool(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_uint(struct flb_json_node *node) +{ + return yyjson_is_uint(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_sint(struct flb_json_node *node) +{ + return yyjson_is_sint(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_int(struct flb_json_node *node) +{ + return yyjson_is_int(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_real(struct flb_json_node *node) +{ + return yyjson_is_real(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_str(struct flb_json_node *node) +{ + return yyjson_is_str(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_arr(struct flb_json_node *node) +{ + return yyjson_is_arr(flb_json_node_val(node)); +} + +static inline int flb_json_node_is_obj(struct flb_json_node *node) +{ + return yyjson_is_obj(flb_json_node_val(node)); +} + +static inline int flb_json_node_get_bool(struct flb_json_node *node) +{ + return yyjson_get_bool(flb_json_node_val(node)); +} + +static inline uint64_t flb_json_node_get_uint(struct flb_json_node *node) +{ + return yyjson_get_uint(flb_json_node_val(node)); +} + +static inline int64_t flb_json_node_get_sint(struct flb_json_node *node) +{ + return yyjson_get_sint(flb_json_node_val(node)); +} + +static inline double flb_json_node_get_real(struct flb_json_node *node) +{ + return yyjson_get_real(flb_json_node_val(node)); +} + +static inline const char *flb_json_node_get_str(struct flb_json_node *node) +{ + return yyjson_get_str(flb_json_node_val(node)); +} + +static inline size_t flb_json_node_get_len(struct flb_json_node *node) +{ + return yyjson_get_len(flb_json_node_val(node)); +} + +/* string and its length, or NULL and a zero length when node is not a string */ +static inline const char *flb_json_node_get_strn(struct flb_json_node *node, + size_t *length) +{ + yyjson_val *value; + + value = flb_json_node_val(node); + if (!yyjson_is_str(value)) { + *length = 0; + return NULL; + } + + *length = unsafe_yyjson_get_len(value); + return unsafe_yyjson_get_str(value); +} + +static inline size_t flb_json_node_arr_size(struct flb_json_node *node) +{ + return yyjson_arr_size(flb_json_node_val(node)); +} + +static inline size_t flb_json_node_obj_size(struct flb_json_node *node) +{ + return yyjson_obj_size(flb_json_node_val(node)); +} + +/* first element of an array, or first key of an object */ +static inline struct flb_json_node *flb_json_node_first(struct flb_json_node *node) +{ + yyjson_val *value; + + value = flb_json_node_val(node); + if ((!yyjson_is_arr(value) && !yyjson_is_obj(value)) || + unsafe_yyjson_get_len(value) == 0) { + return NULL; + } + + return flb_json_node_wrap(unsafe_yyjson_get_first(value)); +} + +/* element or key that follows the given element or object value */ +static inline struct flb_json_node *flb_json_node_next(struct flb_json_node *node) +{ + if (node == NULL) { + return NULL; + } + + /* callers stop on the container size, the last element has no successor */ + return flb_json_node_wrap(unsafe_yyjson_get_next(flb_json_node_val(node))); +} + +/* value of an object key */ +static inline struct flb_json_node *flb_json_node_key_value(struct flb_json_node *key) +{ + if (key == NULL) { + return NULL; + } + + /* in a yyjson tree each object value follows its key */ + return flb_json_node_wrap(flb_json_node_val(key) + 1); +} + +/* value of the first key that matches name ignoring ASCII case, or NULL */ +static inline struct flb_json_node *flb_json_node_obj_get_case(struct flb_json_node *object, + const char *name, + size_t length) +{ + size_t index; + size_t maximum; + yyjson_val *key; + yyjson_val *value; + + if (name == NULL || !yyjson_is_obj(flb_json_node_val(object))) { + return NULL; + } + + yyjson_obj_foreach(flb_json_node_val(object), index, maximum, key, value) { + if (unsafe_yyjson_get_len(key) == length && + strncasecmp(unsafe_yyjson_get_str(key), name, length) == 0) { + return flb_json_node_wrap(value); + } + } + + return NULL; +} + +#else + +/* without a tree backend no tree is read, so no node exists */ +static inline struct flb_json_node *flb_json_tree_root(struct flb_json_tree *tree) +{ + (void) tree; + return NULL; +} + +static inline int flb_json_node_type(struct flb_json_node *node) +{ + (void) node; + return FLB_JSON_NODE_NONE; +} + +static inline int flb_json_node_is_null(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_bool(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_uint(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_sint(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_int(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_real(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_str(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_arr(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_is_obj(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int flb_json_node_get_bool(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline uint64_t flb_json_node_get_uint(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline int64_t flb_json_node_get_sint(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline double flb_json_node_get_real(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline const char *flb_json_node_get_str(struct flb_json_node *node) +{ + (void) node; + return NULL; +} + +static inline size_t flb_json_node_get_len(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline const char *flb_json_node_get_strn(struct flb_json_node *node, + size_t *length) +{ + (void) node; + *length = 0; + return NULL; +} + +static inline size_t flb_json_node_arr_size(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline size_t flb_json_node_obj_size(struct flb_json_node *node) +{ + (void) node; + return 0; +} + +static inline struct flb_json_node *flb_json_node_first(struct flb_json_node *node) +{ + (void) node; + return NULL; +} + +static inline struct flb_json_node *flb_json_node_next(struct flb_json_node *node) +{ + (void) node; + return NULL; +} + +static inline struct flb_json_node *flb_json_node_key_value(struct flb_json_node *key) +{ + (void) key; + return NULL; +} + +static inline struct flb_json_node *flb_json_node_obj_get_case(struct flb_json_node *object, + const char *name, + size_t length) +{ + (void) object; + (void) name; + (void) length; + return NULL; +} + +#endif + +#define flb_json_node_arr_foreach(array, index, maximum, value) \ + for ((index) = 0, (maximum) = flb_json_node_arr_size(array), \ + (value) = flb_json_node_first(array); \ + (index) < (maximum); \ + (index)++, (value) = flb_json_node_next(value)) + +#define flb_json_node_obj_foreach(object, index, maximum, key, value) \ + for ((index) = 0, (maximum) = flb_json_node_obj_size(object), \ + (key) = flb_json_node_first(object), \ + (value) = flb_json_node_key_value(key); \ + (index) < (maximum); \ + (index)++, (key) = flb_json_node_next(value), \ + (value) = flb_json_node_key_value(key)) + +#endif diff --git a/include/fluent-bit/flb_opentelemetry.h b/include/fluent-bit/flb_opentelemetry.h index cdb529e0aef..34bff108765 100644 --- a/include/fluent-bit/flb_opentelemetry.h +++ b/include/fluent-bit/flb_opentelemetry.h @@ -25,6 +25,7 @@ #include #include #include +#include /* Error code values from flb_opentelemetry logs/traces helpers */ #define FLB_OTEL_LOGS_ERR_GENERIC_ERROR -1 @@ -119,6 +120,17 @@ struct flb_opentelemetry_otlp_logs_options { int logs_body_key_attributes; }; +/* Options shared by OTLP/HTTP protobuf log decoders. */ +struct flb_opentelemetry_logs_decode_options { + const char *metadata_key; + const char *body_key; + size_t output_limit; +}; + +struct flb_mp_chunk_cobj; +struct flb_opentelemetry_logs_protobuf_cfl; +struct flb_opentelemetry_logs_json_cfl; + /* Backward-compatible alias for older external callers. */ #define flb_opentelemetry_otlp_json_options flb_opentelemetry_otlp_logs_options @@ -204,6 +216,60 @@ int flb_opentelemetry_logs_json_to_msgpack(struct flb_log_event_encoder *encoder const char *logs_body_key, int *error_status); +/* + * Returned by flb_opentelemetry_logs_json_to_cfl() for a payload it leaves to + * flb_opentelemetry_logs_json_to_msgpack(): records CFL cannot hold, such as a + * map key that is not a string, and payloads the JSON tree does not read, + * such as content after the request object or a build without yyjson. + */ +#define FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED -2 + +int flb_opentelemetry_logs_json_to_cfl( + const char *body, size_t len, + const struct flb_opentelemetry_logs_decode_options *options, + struct flb_opentelemetry_logs_json_cfl **result, + int *error_status); + +struct flb_mp_chunk_cobj *flb_opentelemetry_logs_json_cfl_take_chunk( + struct flb_opentelemetry_logs_json_cfl *result); + +size_t flb_opentelemetry_logs_json_cfl_get_encoded_size( + struct flb_opentelemetry_logs_json_cfl *result); + +void flb_opentelemetry_logs_json_cfl_destroy( + struct flb_opentelemetry_logs_json_cfl *result); + +int flb_opentelemetry_protobuf_validate( + const ProtobufCMessageDescriptor *descriptor, + const void *data, size_t size); + +int flb_opentelemetry_logs_protobuf_to_msgpack( + struct flb_log_event_encoder *encoder, + const void *payload, size_t payload_size, + const struct flb_opentelemetry_logs_decode_options *options, + size_t *record_count); + +int flb_opentelemetry_logs_protobuf_to_cfl( + const void *payload, size_t payload_size, + const struct flb_opentelemetry_logs_decode_options *options, + struct flb_opentelemetry_logs_protobuf_cfl **result); + +/* + * The returned result owns both the decoded protobuf arena and the CFL chunk. + * Values in the CFL chunk can borrow strings and byte arrays from that arena. + * After taking the chunk, the caller must consume or destroy it synchronously + * before destroying the result. Neither object can cross an asynchronous + * queue boundary; only the final owned MessagePack buffer may do so. + */ +struct flb_mp_chunk_cobj *flb_opentelemetry_logs_protobuf_cfl_take_chunk( + struct flb_opentelemetry_logs_protobuf_cfl *result); + +size_t flb_opentelemetry_logs_protobuf_cfl_get_encoded_size( + struct flb_opentelemetry_logs_protobuf_cfl *result); + +void flb_opentelemetry_logs_protobuf_cfl_destroy( + struct flb_opentelemetry_logs_protobuf_cfl *result); + int flb_opentelemetry_metrics_json_to_cmt(struct cfl_list *context_list, const char *body, size_t len); diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index 6d0b6eab6ea..d177052eca1 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -65,8 +65,15 @@ * FLB_PROCESSOR_FLAG_LOG_WINDOWS: the logs callback only depends on each * record and its group, so the processor chain can run it over large buffers * in group aligned windows. + * + * FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE: the logs callback is safe to invoke + * from an input protocol worker. It must not depend on the caller's event + * loop, scheduler, coroutine or thread-local state, emit records through + * thread-bound APIs, or require ordering between independently received + * batches. */ #define FLB_PROCESSOR_FLAG_LOG_WINDOWS 1 +#define FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE (1 << 1) /* The current values mean the processor stack will @@ -89,6 +96,7 @@ struct flb_log_event_decoder; struct flb_log_event_encoder; struct flb_processor_instance; struct flb_condition; +struct flb_mp_chunk_cobj; struct flb_processor_unit { int event_type; @@ -276,6 +284,16 @@ int flb_processor_run_with_count(struct flb_processor *proc, size_t starting_sta void **out_buf, size_t *out_size, int input_records, int *output_records); +/* Run the logs processors over a chunk of CFL records, consuming the chunk */ +int flb_processor_logs_chunk_supported(struct flb_processor *proc, + int worker_thread); +int flb_processor_run_logs_chunk(struct flb_processor *proc, + const char *tag, size_t tag_len, + struct flb_mp_chunk_cobj *chunk, + int worker_thread, + void **out_buf, size_t *out_size, + int *output_records); + struct flb_processor_unit *flb_processor_unit_create(struct flb_processor *proc, int event_type, char *unit_name); diff --git a/include/fluent-bit/http_server/flb_http_server.h b/include/fluent-bit/http_server/flb_http_server.h index 07f4bc98f4f..d65e4729e19 100755 --- a/include/fluent-bit/http_server/flb_http_server.h +++ b/include/fluent-bit/http_server/flb_http_server.h @@ -103,6 +103,9 @@ struct flb_http_server_options { flb_http_server_worker_callback cb_worker_exit; }; +struct flb_http_server_balancer; +struct flb_http_server_handoff; + struct flb_http_server { /* Internal */ struct mk_event listener_event; @@ -138,6 +141,9 @@ struct flb_http_server { flb_http_server_worker_callback cb_worker_init; flb_http_server_worker_callback cb_worker_exit; struct flb_downstream_worker_runtime *runtime; + /* connection balancing between managed workers */ + struct flb_http_server_balancer *balancer; + struct flb_http_server_handoff *handoff; }; struct flb_http_server_session { @@ -156,6 +162,7 @@ struct flb_http_server_session { int drop_pending; int destroying; int connection_slot_reserved; + int balanced_connection; struct flb_connection *connection; struct flb_http_server *parent; diff --git a/plugins/in_opentelemetry/CMakeLists.txt b/plugins/in_opentelemetry/CMakeLists.txt index 543d9efd13f..5b31b406906 100644 --- a/plugins/in_opentelemetry/CMakeLists.txt +++ b/plugins/in_opentelemetry/CMakeLists.txt @@ -4,7 +4,6 @@ endif() set(src opentelemetry.c - opentelemetry_protobuf.c opentelemetry_prot.c opentelemetry_logs.c opentelemetry_traces.c diff --git a/plugins/in_opentelemetry/opentelemetry_logs.c b/plugins/in_opentelemetry/opentelemetry_logs.c index 52dc92dab38..bbee1a38576 100644 --- a/plugins/in_opentelemetry/opentelemetry_logs.c +++ b/plugins/in_opentelemetry/opentelemetry_logs.c @@ -20,776 +20,59 @@ #include #include #include -#include #include -#include +#include +#include #include -#include -#include - -#include "opentelemetry_protobuf.h" #include "opentelemetry.h" #include "opentelemetry_logs.h" #include "opentelemetry_utils.h" /* - * Small requests use protobuf-c's default allocator to avoid reserving an - * arena chunk. Realistic batched requests use the arena to amortize the large - * number of short-lived allocations performed by protobuf-c. - */ -#define OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE 8192 -#define OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE 4096 -#define OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE 65536 - -static void *protobuf_arena_chunk_malloc(void *context, size_t size) -{ - (void) context; - - return flb_malloc(size); -} - -static void protobuf_arena_chunk_free(void *context, void *pointer) -{ - (void) context; - - flb_free(pointer); -} - -static void *protobuf_arena_alloc(void *allocator_data, size_t size) -{ - struct cfl_arena *arena; - - arena = allocator_data; - if (size == 0) { - size = 1; - } - - return cfl_arena_malloc(arena, size); -} - -static void protobuf_arena_free(void *allocator_data, void *pointer) -{ - (void) allocator_data; - (void) pointer; -} - -static struct cfl_arena *protobuf_arena_create(void) -{ - struct cfl_arena_options options; - - cfl_arena_options_init(&options); - options.chunk_size = OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE; - options.maximum_chunk_size = OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE; - options.malloc_fn = protobuf_arena_chunk_malloc; - options.free_fn = protobuf_arena_chunk_free; - - return cfl_arena_create_with_options(&options); -} - -static Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest * -protobuf_logs_unpack(ProtobufCAllocator *allocator, size_t size, const uint8_t *data) -{ - if (opentelemetry_protobuf_validate( - &opentelemetry__proto__collector__logs__v1__export_logs_service_request__descriptor, - data, size) != 0) { - return NULL; - } - return opentelemetry__proto__collector__logs__v1__export_logs_service_request__unpack( - allocator, size, data); -} - -static void protobuf_logs_free( - Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *logs, - ProtobufCAllocator *allocator) -{ - opentelemetry__proto__collector__logs__v1__export_logs_service_request__free_unpacked( - logs, allocator); -} - -/* - * OTLP encoding functions to pack the log records as msgpack - * ---------------------------------------------------------- + * Encode JSON logs through the single-pass CFL decoder. The generic JSON to + * msgpack path parses the payload into msgpack first and then walks that + * msgpack again to build the records, it is only used for the rare payloads + * CFL cannot hold. */ -static int otlp_pack_any_value(msgpack_packer *mp_pck, Opentelemetry__Proto__Common__V1__AnyValue *body); - -static int otel_pack_string(msgpack_packer *mp_pck, char *str) +static int opentelemetry_logs_json_encode_direct(struct flb_opentelemetry *ctx, + struct flb_log_event_encoder *encoder, + const char *payload, size_t size, + struct flb_opentelemetry_logs_decode_options *options, + size_t *record_count, + int *error_status) { - return msgpack_pack_str_with_body(mp_pck, str, strlen(str)); -} - -static int otel_pack_bool(msgpack_packer *mp_pck, bool val) -{ - if (val) { - return msgpack_pack_true(mp_pck); - } - else { - return msgpack_pack_false(mp_pck); - } -} - -static int otel_pack_int(msgpack_packer *mp_pck, int64_t val) -{ - return msgpack_pack_int64(mp_pck, val); -} - -static int otel_pack_double(msgpack_packer *mp_pck, double val) -{ - return msgpack_pack_double(mp_pck, val); -} - -static int otel_pack_kvarray(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__KeyValue **kv_array, - size_t kv_count) -{ - int result; - int index; - - result = msgpack_pack_map(mp_pck, kv_count); - - if (result != 0) { - return result; - } - - for (index = 0; index < kv_count && result == 0; index++) { - result = otel_pack_string(mp_pck, kv_array[index]->key); - - if(result == 0) { - result = otlp_pack_any_value(mp_pck, kv_array[index]->value); - } - } - - return result; -} - -static int otel_pack_kvlist(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__KeyValueList *kv_list) -{ - int kv_index; int ret; - char *key; - Opentelemetry__Proto__Common__V1__AnyValue *value; - - ret = msgpack_pack_map(mp_pck, kv_list->n_values); - if (ret != 0) { - return ret; - } - - for (kv_index = 0; kv_index < kv_list->n_values && ret == 0; kv_index++) { - key = kv_list->values[kv_index]->key; - value = kv_list->values[kv_index]->value; + struct flb_mp_chunk_cobj *chunk; + struct flb_opentelemetry_logs_json_cfl *result; - ret = otel_pack_string(mp_pck, key); - - if(ret == 0) { - ret = otlp_pack_any_value(mp_pck, value); - } + result = NULL; + ret = flb_opentelemetry_logs_json_to_cfl(payload, size, options, &result, error_status); + if (ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED) { + return flb_opentelemetry_logs_json_to_msgpack(encoder, payload, size, + ctx->logs_body_key, error_status); } - - return ret; -} - -static int otel_pack_array(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__ArrayValue *array) -{ - int ret; - int array_index; - - ret = msgpack_pack_array(mp_pck, array->n_values); - if (ret != 0) { - return ret; - } - - for (array_index = 0; array_index < array->n_values && ret == 0; array_index++) { - ret = otlp_pack_any_value(mp_pck, array->values[array_index]); - } - - return ret; -} - -static int otel_pack_bytes(msgpack_packer *mp_pck, - ProtobufCBinaryData bytes) -{ - return msgpack_pack_bin_with_body(mp_pck, bytes.data, bytes.len); -} - -static int otlp_pack_any_value(msgpack_packer *mp_pck, - Opentelemetry__Proto__Common__V1__AnyValue *body) -{ - int result; - - result = -2; - - if (body == NULL) { - msgpack_pack_nil(mp_pck); - return 0; - } - - switch(body->value_case){ - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: - result = otel_pack_string(mp_pck, body->string_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE_STRINDEX: - /* Profiling-only string dictionary reference: ignore in logs. */ - result = msgpack_pack_nil(mp_pck); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: - result = otel_pack_bool(mp_pck, body->bool_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: - result = otel_pack_int(mp_pck, body->int_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: - result = otel_pack_double(mp_pck, body->double_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: - result = otel_pack_array(mp_pck, body->array_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: - result = otel_pack_kvlist(mp_pck, body->kvlist_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: - result = otel_pack_bytes(mp_pck, body->bytes_value); - break; - - case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET: - /* treat an unset value as null */ - result = msgpack_pack_nil(mp_pck); - break; - - default: - break; - } - - if (result == -2) { - flb_error("[otel]: invalid value type in pack_any_value"); - result = -1; - } - - return result; -} - -/* https://opentelemetry.io/docs/specs/otel/logs/data-model/#log-and-event-record-definition */ -static int otel_pack_v1_metadata(struct flb_opentelemetry *ctx, - msgpack_packer *mp_pck, - struct Opentelemetry__Proto__Logs__V1__LogRecord *log_record, - Opentelemetry__Proto__Resource__V1__Resource *resource, - Opentelemetry__Proto__Common__V1__InstrumentationScope *scope) -{ - int ret; - int len; - struct flb_mp_map_header mh; - struct flb_mp_map_header otlp_mh; - - flb_mp_map_header_init(&otlp_mh, mp_pck); - - len = flb_sds_len(ctx->logs_metadata_key); - - /* otlp key start */ - flb_mp_map_header_append(&otlp_mh); - - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, ctx->logs_metadata_key, len); - - flb_mp_map_header_init(&mh, mp_pck); - - if (log_record->observed_time_unix_nano != 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 18); - msgpack_pack_str_body(mp_pck, "observed_timestamp", 18); - msgpack_pack_uint64(mp_pck, log_record->observed_time_unix_nano); - } - - /* Value of 0 indicates unknown or missing timestamp. */ - if (log_record->time_unix_nano != 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 9); - msgpack_pack_str_body(mp_pck, "timestamp", 9); - msgpack_pack_uint64(mp_pck, log_record->time_unix_nano); - } - - /* https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber */ - if (log_record->severity_number >= 1 && log_record->severity_number <= 24) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 15); - msgpack_pack_str_body(mp_pck, "severity_number", 15); - msgpack_pack_uint64(mp_pck, log_record->severity_number); - } - - if (log_record->severity_text != NULL && strlen(log_record->severity_text) > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 13); - msgpack_pack_str_body(mp_pck, "severity_text", 13); - msgpack_pack_str(mp_pck, strlen(log_record->severity_text)); - msgpack_pack_str_body(mp_pck, log_record->severity_text, strlen(log_record->severity_text)); - } - - if (log_record->n_attributes > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "attributes", 10); - ret = otel_pack_kvarray(mp_pck, - log_record->attributes, - log_record->n_attributes); - if (ret != 0) { - return ret; - } - } - - if (log_record->dropped_attributes_count > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 24); - msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); - msgpack_pack_uint64(mp_pck, log_record->dropped_attributes_count); - } - - if (log_record->trace_id.len > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 8); - msgpack_pack_str_body(mp_pck, "trace_id", 8); - ret = otel_pack_bytes(mp_pck, log_record->trace_id); - if (ret != 0) { - return ret; - } - } - - if (log_record->span_id.len > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "span_id", 7); - ret = otel_pack_bytes(mp_pck, log_record->span_id); - if (ret != 0) { - return ret; - } - } - - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 11); - msgpack_pack_str_body(mp_pck, "trace_flags", 11); - msgpack_pack_uint8(mp_pck, (uint8_t) log_record->flags & 0xff); - - if (log_record->event_name != NULL && strlen(log_record->event_name) > 0) { - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "event_name", 10); - msgpack_pack_str(mp_pck, strlen(log_record->event_name)); - msgpack_pack_str_body(mp_pck, log_record->event_name, strlen(log_record->event_name)); - } - - flb_mp_map_header_end(&mh); - - /* otlp key end */ - flb_mp_map_header_end(&otlp_mh); - - return 0; -} - -static int binary_payload_to_msgpack(struct flb_opentelemetry *ctx, - struct flb_log_event_encoder *encoder, - char *tag, size_t tag_len, - uint8_t *in_buf, - size_t in_size, - size_t *record_count) -{ - int ret = 0; - int len; - int resource_logs_index; - int scope_log_index; - int log_record_index; - char *logs_body_key; - int scope_has_schema_url; - struct cfl_arena *protobuf_arena; - struct flb_mp_map_header mh; - struct flb_mp_map_header mh_tmp; - struct flb_time tm; - ProtobufCAllocator arena_allocator; - ProtobufCAllocator *protobuf_allocator; - - msgpack_packer *mp_pck; - msgpack_packer *mp_pck_meta; - - /* OTel proto suff */ - Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *input_logs; - Opentelemetry__Proto__Logs__V1__ScopeLogs **scope_logs; - Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; - Opentelemetry__Proto__Common__V1__InstrumentationScope *scope; - - Opentelemetry__Proto__Logs__V1__ResourceLogs **resource_logs; - Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; - Opentelemetry__Proto__Logs__V1__LogRecord **log_records; - Opentelemetry__Proto__Resource__V1__Resource *resource; - - mp_pck = &encoder->body.packer; - mp_pck_meta = &encoder->metadata.packer; - input_logs = NULL; - protobuf_arena = NULL; - protobuf_allocator = NULL; - *record_count = 0; - - if (in_size >= OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE) { - protobuf_arena = protobuf_arena_create(); - if (protobuf_arena != NULL) { - arena_allocator.alloc = protobuf_arena_alloc; - arena_allocator.free = protobuf_arena_free; - arena_allocator.allocator_data = protobuf_arena; - protobuf_allocator = &arena_allocator; - } - } - - /* unpack logs from protobuf payload */ - input_logs = protobuf_logs_unpack(protobuf_allocator, in_size, in_buf); - if (input_logs == NULL) { - flb_plg_warn(ctx->ins, "failed to unpack input logs from OpenTelemetry payload"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - resource_logs = input_logs->resource_logs; - if (input_logs->n_resource_logs == 0) { - ret = 0; - goto binary_payload_to_msgpack_end; - } - - if (resource_logs == NULL) { - flb_plg_warn(ctx->ins, "no resource logs found"); - ret = -1; - goto binary_payload_to_msgpack_end; + flb_opentelemetry_logs_json_cfl_destroy(result); + return -1; } - for (resource_logs_index = 0; resource_logs_index < input_logs->n_resource_logs; resource_logs_index++) { - resource_log = resource_logs[resource_logs_index]; - if (resource_log == NULL) { - flb_plg_warn(ctx->ins, "null resource logs entry found"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - resource = resource_log->resource; - scope_logs = resource_log->scope_logs; - - if (resource_log->n_scope_logs > 0 && scope_logs == NULL) { - flb_plg_warn(ctx->ins, "no scope logs found"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - for (scope_log_index = 0; scope_log_index < resource_log->n_scope_logs; scope_log_index++) { - scope_log = scope_logs[scope_log_index]; - if (scope_log == NULL) { - flb_plg_warn(ctx->ins, "null scope logs entry found"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - log_records = scope_log->log_records; - - if (scope_log->n_log_records == 0) { - continue; - } - - if (log_records == NULL) { - flb_plg_warn(ctx->ins, "no log records found"); - ret = -1; - goto binary_payload_to_msgpack_end; - } - - flb_log_event_encoder_group_init(encoder); - - /* pack schema (internal) */ - ret = flb_log_event_encoder_append_metadata_values(encoder, - FLB_LOG_EVENT_STRING_VALUE("schema", 6), - FLB_LOG_EVENT_STRING_VALUE("otlp", 4), - FLB_LOG_EVENT_STRING_VALUE("resource_id", 11), - FLB_LOG_EVENT_INT64_VALUE(resource_logs_index), - FLB_LOG_EVENT_STRING_VALUE("scope_id", 8), - FLB_LOG_EVENT_INT64_VALUE(scope_log_index)); - - - ret = flb_log_event_encoder_dynamic_field_reset(&encoder->body); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "failed to reset log event body: %s", - flb_log_event_encoder_get_error_description(ret)); - goto binary_payload_to_msgpack_end; - } - - flb_mp_map_header_init(&mh, mp_pck); - - /* Resource */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 8); - msgpack_pack_str_body(mp_pck, "resource", 8); - - flb_mp_map_header_init(&mh_tmp, mp_pck); - if (resource) { - /* look for OTel resource attributes */ - if (resource->n_attributes > 0 && resource->attributes) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "attributes", 10); - - ret = otel_pack_kvarray(mp_pck, - resource->attributes, - resource->n_attributes); - if (ret != 0) { - goto binary_payload_to_msgpack_end; - } - } - - if (resource->dropped_attributes_count > 0) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 24); - msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); - msgpack_pack_uint64(mp_pck, resource->dropped_attributes_count); - } - - if (resource_log->schema_url) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "schema_url", 10); - - len = strlen(resource_log->schema_url); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, resource_log->schema_url, len); - } - } - flb_mp_map_header_end(&mh_tmp); - - /* scope */ - flb_mp_map_header_append(&mh); - msgpack_pack_str(mp_pck, 5); - msgpack_pack_str_body(mp_pck, "scope", 5); - - /* Scope */ - scope = scope_log->scope; - scope_has_schema_url = FLB_FALSE; - - if (scope_log->schema_url && strlen(scope_log->schema_url) > 0) { - scope_has_schema_url = FLB_TRUE; - } - - if (scope && (scope->name || scope->version || - scope->n_attributes > 0 || scope->dropped_attributes_count > 0 || - scope_has_schema_url == FLB_TRUE)) { - flb_mp_map_header_init(&mh_tmp, mp_pck); - - if (scope_has_schema_url == FLB_TRUE) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "schema_url", 10); - - len = strlen(scope_log->schema_url); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, scope_log->schema_url, len); - } - - if (scope->name && strlen(scope->name) > 0) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 4); - msgpack_pack_str_body(mp_pck, "name", 4); - - len = strlen(scope->name); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, scope->name, len); - } - if (scope->version && strlen(scope->version) > 0) { - flb_mp_map_header_append(&mh_tmp); - - msgpack_pack_str(mp_pck, 7); - msgpack_pack_str_body(mp_pck, "version", 7); - - len = strlen(scope->version); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, scope->version, len); - } - - if (scope->n_attributes > 0 && scope->attributes) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "attributes", 10); - ret = otel_pack_kvarray(mp_pck, - scope->attributes, - scope->n_attributes); - if (ret != 0) { - goto binary_payload_to_msgpack_end; - } - } - - if (scope->dropped_attributes_count > 0) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 24); - msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); - msgpack_pack_uint64(mp_pck, scope->dropped_attributes_count); - } - - flb_mp_map_header_end(&mh_tmp); - } - else { - flb_mp_map_header_init(&mh_tmp, mp_pck); - - if (scope_has_schema_url == FLB_TRUE) { - flb_mp_map_header_append(&mh_tmp); - msgpack_pack_str(mp_pck, 10); - msgpack_pack_str_body(mp_pck, "schema_url", 10); - - len = strlen(scope_log->schema_url); - msgpack_pack_str(mp_pck, len); - msgpack_pack_str_body(mp_pck, scope_log->schema_url, len); - } - - flb_mp_map_header_end(&mh_tmp); - } - - flb_mp_map_header_end(&mh); - - ret = flb_log_event_encoder_dynamic_field_flush(&encoder->body); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "could not set group content metadata: %s", - flb_log_event_encoder_get_error_description(ret)); - goto binary_payload_to_msgpack_end; - } - - ret = flb_log_event_encoder_group_header_end(encoder); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - goto binary_payload_to_msgpack_end; - } - - for (log_record_index=0; log_record_index < scope_log->n_log_records; log_record_index++) { - ret = flb_log_event_encoder_begin_record(encoder); - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - if (log_records[log_record_index]->time_unix_nano > 0) { - ret = flb_time_from_uint64( - &tm, - log_records[log_record_index]->time_unix_nano); - if (ret == 0) { - ret = flb_log_event_encoder_set_timestamp(encoder, &tm); - } - } - else if (log_records[log_record_index]->observed_time_unix_nano > 0) { - ret = flb_time_from_uint64( - &tm, - log_records[log_record_index]->observed_time_unix_nano); - if (ret == 0) { - ret = flb_log_event_encoder_set_timestamp(encoder, &tm); - } - } - else { - flb_time_get(&tm); - ret = flb_log_event_encoder_set_timestamp(encoder, &tm); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_dynamic_field_reset(&encoder->metadata); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "failed to reset log event metadata: %s", - flb_log_event_encoder_get_error_description(ret)); - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - ret = otel_pack_v1_metadata(ctx, - mp_pck_meta, - log_records[log_record_index], - resource, - scope_log->scope); - } - - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to convert log record"); - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - ret = flb_log_event_encoder_dynamic_field_flush(&encoder->metadata); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_dynamic_field_reset(&encoder->body); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - flb_plg_error(ctx->ins, "failed to reset log event body: %s", - flb_log_event_encoder_get_error_description(ret)); - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else if (ctx->logs_body_key == NULL && - log_records[log_record_index]->body != NULL && - log_records[log_record_index]->body->value_case == - OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { - ret = otlp_pack_any_value( - mp_pck, - log_records[log_record_index]->body); - } - else { - logs_body_key = ctx->logs_body_key; - if (logs_body_key == NULL) { - logs_body_key = "log"; - } - ret = msgpack_pack_map(mp_pck, 1); - if (ret == 0) { - ret = msgpack_pack_str(mp_pck, strlen(logs_body_key)); - } - if (ret == 0) { - ret = msgpack_pack_str_body(mp_pck, - logs_body_key, - strlen(logs_body_key)); - } - if (ret == 0) { - ret = otlp_pack_any_value( - mp_pck, - log_records[log_record_index]->body); - } - } - - if (ret != 0) { - flb_plg_error(ctx->ins, "failed to convert log record body"); - ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; - } - else { - ret = flb_log_event_encoder_dynamic_field_flush(&encoder->body); - } - } - - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - ret = flb_log_event_encoder_commit_record(encoder); - if (ret == FLB_EVENT_ENCODER_SUCCESS) { - (*record_count)++; - } - else { - goto binary_payload_to_msgpack_end; - } - } - else { - flb_plg_error(ctx->ins, "marshalling error"); - goto binary_payload_to_msgpack_end; - } - } - - ret = flb_log_event_encoder_group_end(encoder); - if (ret != FLB_EVENT_ENCODER_SUCCESS) { - goto binary_payload_to_msgpack_end; - } - - } + chunk = flb_opentelemetry_logs_json_cfl_take_chunk(result); + if (chunk == NULL) { + flb_opentelemetry_logs_json_cfl_destroy(result); + return -1; } - binary_payload_to_msgpack_end: - if (input_logs) { - protobuf_logs_free(input_logs, protobuf_allocator); - } - cfl_arena_destroy(protobuf_arena); + /* the records borrow strings from the JSON document owned by 'result' */ + chunk->log_encoder = encoder; + ret = flb_mp_chunk_cobj_encode_append(chunk); + *record_count = flb_mp_chunk_cobj_count_log_records(chunk); + chunk->log_encoder = NULL; - if (ret != 0) { - return -1; - } + flb_mp_chunk_cobj_destroy(chunk); + flb_opentelemetry_logs_json_cfl_destroy(result); - return 0; + return ret; } /* @@ -822,11 +105,22 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, uint64_t payload_size; size_t record_count; struct flb_log_event_encoder *encoder; + struct flb_mp_chunk_cobj *chunk; + struct flb_opentelemetry_logs_decode_options decode_options; + struct flb_opentelemetry_logs_protobuf_cfl *cfl_result; + struct flb_opentelemetry_logs_json_cfl *json_cfl_result; + int worker_thread; buf = (char *) data; payload = data; payload_size = size; record_count = 0; + cfl_result = NULL; + json_cfl_result = NULL; + worker_thread = opentelemetry_uses_worker_ingress_queue(ctx); + decode_options.metadata_key = ctx->logs_metadata_key; + decode_options.body_key = ctx->logs_body_key; + decode_options.output_limit = *remaining; /* Detect the type of payload */ if (content_type) { @@ -846,6 +140,66 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, } } + /* + * JSON and protobuf records go straight to input processors as CFL records + * when they can take them. Workers enqueue the serialized processor + * output as already processed, so the engine does not run the chain twice. + * A JSON payload CFL cannot hold takes the msgpack path below. + */ + if (flb_input_log_chunk_supported(ctx->ins, worker_thread) == FLB_TRUE) { + if (is_proto == FLB_TRUE) { + ret = flb_opentelemetry_logs_protobuf_to_cfl( + payload, payload_size, &decode_options, &cfl_result); + if (ret == 0) { + *remaining -= flb_opentelemetry_logs_protobuf_cfl_get_encoded_size(cfl_result); + chunk = flb_opentelemetry_logs_protobuf_cfl_take_chunk(cfl_result); + } + } + else { + ret = flb_opentelemetry_logs_json_to_cfl( + (const char *) payload, payload_size, &decode_options, + &json_cfl_result, &error_status); + if (ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED) { + goto msgpack_path; + } + if (ret == 0) { + *remaining -= flb_opentelemetry_logs_json_cfl_get_encoded_size(json_cfl_result); + chunk = flb_opentelemetry_logs_json_cfl_take_chunk(json_cfl_result); + } + } + if (ret != 0) { + flb_plg_error(ctx->ins, "failed to process logs from %s payload (%i) %s", + is_proto == FLB_TRUE ? "protobuf" : "JSON", error_status, + flb_opentelemetry_error_to_string(error_status)); + flb_opentelemetry_logs_protobuf_cfl_destroy(cfl_result); + flb_opentelemetry_logs_json_cfl_destroy(json_cfl_result); + return -1; + } + + /* like the Collector, a request without log records is accepted */ + if (cfl_list_is_empty(&chunk->records)) { + flb_mp_chunk_cobj_destroy(chunk); + ret = 0; + } + else { + ret = flb_input_log_append_chunk(ctx->ins, tag, flb_sds_len(tag), + chunk, worker_thread); + } + flb_opentelemetry_logs_protobuf_cfl_destroy(cfl_result); + flb_opentelemetry_logs_json_cfl_destroy(json_cfl_result); + if (ret == FLB_INPUT_INGRESS_BUSY) { + /* the client retries once the ingress queue has room */ + return FLB_INPUT_INGRESS_BUSY; + } + else if (ret != 0) { + flb_plg_error(ctx->ins, "failed to append logs to the input buffer"); + return -1; + } + + return 0; + } + +msgpack_path: encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); if (encoder == NULL) { return -1; @@ -853,31 +207,31 @@ int opentelemetry_process_logs_with_budget(struct flb_opentelemetry *ctx, encoder->output_limit = *remaining; if (is_proto == FLB_TRUE) { - ret = binary_payload_to_msgpack(ctx, encoder, - tag, tag_len, - (uint8_t *) payload, payload_size, - &record_count); + ret = flb_opentelemetry_logs_protobuf_to_msgpack( + encoder, payload, payload_size, &decode_options, &record_count); if (ret < 0) { flb_plg_error(ctx->ins, "failed to process logs from protobuf payload"); } } else { - ret = flb_opentelemetry_logs_json_to_msgpack(encoder, - (const char *) payload, payload_size, - ctx->logs_body_key, - &error_status); + ret = opentelemetry_logs_json_encode_direct(ctx, encoder, + (const char *) payload, payload_size, + &decode_options, &record_count, + &error_status); if (ret != 0) { - /* we are printing the error for now, let's see what is the user's preference later */ flb_plg_error(ctx->ins, "failed to process logs from JSON payload (%i) %s", error_status, flb_opentelemetry_error_to_string(error_status)); } - } - if (ret >= 0) { + if (ret >= 0 && encoder->output_length == 0) { + /* like the Collector, a request without log records is accepted */ + ret = 0; + } + else if (ret >= 0) { *remaining -= encoder->output_length; - if (opentelemetry_uses_worker_ingress_queue(ctx)) { + if (worker_thread == FLB_TRUE) { size_t allocation_size; void *resized_buffer; diff --git a/plugins/in_opentelemetry/opentelemetry_protobuf.h b/plugins/in_opentelemetry/opentelemetry_protobuf.h deleted file mode 100644 index 5f3e39ede59..00000000000 --- a/plugins/in_opentelemetry/opentelemetry_protobuf.h +++ /dev/null @@ -1,31 +0,0 @@ -/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ - -/* Fluent Bit - * ========== - * Copyright (C) 2015-2026 The Fluent Bit Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef FLB_IN_OPENTELEMETRY_PROTOBUF_H -#define FLB_IN_OPENTELEMETRY_PROTOBUF_H - -#include - -/* Includes the export request and intermediate protobuf message wrappers. */ -#define OTEL_PROTOBUF_MAX_DEPTH 100 - -int opentelemetry_protobuf_validate(const ProtobufCMessageDescriptor *descriptor, - const void *data, size_t size); - -#endif diff --git a/plugins/processor_content_modifier/cm.c b/plugins/processor_content_modifier/cm.c index 5482256b9ef..db743ad05e6 100644 --- a/plugins/processor_content_modifier/cm.c +++ b/plugins/processor_content_modifier/cm.c @@ -169,5 +169,6 @@ struct flb_processor_plugin processor_content_modifier_plugin = { .cb_process_traces = cb_process_traces, .cb_exit = cb_exit, .config_map = config_map, - .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS + .flags = FLB_PROCESSOR_FLAG_LOG_WINDOWS | + FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE }; diff --git a/plugins/processor_content_modifier/cm_logs.c b/plugins/processor_content_modifier/cm_logs.c index 83029e3d71e..3c2a4c1ac61 100644 --- a/plugins/processor_content_modifier/cm_logs.c +++ b/plugins/processor_content_modifier/cm_logs.c @@ -253,6 +253,7 @@ static int run_action_convert(struct content_modifier_ctx *ctx, struct cfl_kvpair *kvpair; struct cfl_variant *v; struct cfl_variant *converted; + struct cfl_variant *arena_converted; /* if the kv pair already exists, remove it from the list */ kvpair = cfl_object_kvpair_get(obj, key); @@ -267,10 +268,50 @@ static int run_action_convert(struct content_modifier_ctx *ctx, return -1; } + kvlist = obj->variant->data.as_kvlist; + if (kvlist->arena != NULL) { + arena_converted = NULL; + + if (converted->type == CFL_VARIANT_STRING) { + arena_converted = cfl_variant_create_from_string_s_in( + kvlist->arena, converted->data.as_string, + cfl_variant_size_get(converted), CFL_FALSE); + } + else if (converted->type == CFL_VARIANT_BYTES) { + arena_converted = cfl_variant_create_from_bytes_in( + kvlist->arena, converted->data.as_bytes, + cfl_variant_size_get(converted), CFL_FALSE); + } + else if (converted->type == CFL_VARIANT_BOOL) { + arena_converted = cfl_variant_create_from_bool_in( + kvlist->arena, converted->data.as_bool); + } + else if (converted->type == CFL_VARIANT_INT) { + arena_converted = cfl_variant_create_from_int64_in( + kvlist->arena, converted->data.as_int64); + } + else if (converted->type == CFL_VARIANT_UINT) { + arena_converted = cfl_variant_create_from_uint64_in( + kvlist->arena, converted->data.as_uint64); + } + else if (converted->type == CFL_VARIANT_DOUBLE) { + arena_converted = cfl_variant_create_from_double_in( + kvlist->arena, converted->data.as_double); + } + else if (converted->type == CFL_VARIANT_NULL) { + arena_converted = cfl_variant_create_from_null_in(kvlist->arena); + } + + cfl_variant_destroy(converted); + if (arena_converted == NULL) { + return -1; + } + converted = arena_converted; + } + /* remove the old kvpair */ cfl_kvpair_destroy(kvpair); - kvlist = obj->variant->data.as_kvlist; ret = cfl_kvlist_insert_s(kvlist, key, cfl_sds_len(key), converted); if (ret != 0) { cfl_variant_destroy(converted); @@ -326,7 +367,7 @@ static struct cfl_variant *otel_get_log_attributes(struct flb_mp_chunk_record *r } } else { - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } diff --git a/plugins/processor_content_modifier/cm_opentelemetry.c b/plugins/processor_content_modifier/cm_opentelemetry.c index 3cb3b60bd80..792793ea114 100644 --- a/plugins/processor_content_modifier/cm_opentelemetry.c +++ b/plugins/processor_content_modifier/cm_opentelemetry.c @@ -59,7 +59,7 @@ struct cfl_variant *cm_otel_get_or_create_attributes(struct cfl_kvlist *kvlist) } /* create an empty kvlist as the value of attributes */ - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } @@ -106,7 +106,7 @@ static struct cfl_variant *otel_get_or_create_attributes(struct cfl_kvlist *kvli } /* create an empty kvlist as the value of attributes */ - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } @@ -188,7 +188,7 @@ static struct cfl_variant *otel_get_or_create_scope_metadata(int telemetry_type, } /* metadata don't exists, create it */ - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } @@ -237,7 +237,7 @@ struct cfl_variant *cm_otel_get_scope_metadata(int telemetry_type, struct cfl_kv } else { /* create "scope" inside kvlist */ - kvlist_tmp = cfl_kvlist_create(); + kvlist_tmp = cfl_kvlist_create_like(kvlist); if (!kvlist_tmp) { return NULL; } diff --git a/plugins/processor_sql/sql.c b/plugins/processor_sql/sql.c index aa3b1c99d22..e7f4619656f 100644 --- a/plugins/processor_sql/sql.c +++ b/plugins/processor_sql/sql.c @@ -709,6 +709,6 @@ struct flb_processor_plugin processor_sql_plugin = { .cb_process_traces = NULL, .cb_exit = cb_exit, .config_map = config_map, - .flags = 0 + .flags = FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE }; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1d0fa9dbde1..324ae6088a5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -128,7 +128,10 @@ endif() set(src ${src} opentelemetry/flb_opentelemetry_logs.c + opentelemetry/flb_opentelemetry_logs_json.c + opentelemetry/flb_opentelemetry_logs_proto.c opentelemetry/flb_opentelemetry_metrics.c + opentelemetry/flb_opentelemetry_protobuf.c opentelemetry/flb_opentelemetry_otlp_json.c opentelemetry/flb_opentelemetry_otlp_proto.c opentelemetry/flb_opentelemetry_traces.c diff --git a/src/flb_downstream.c b/src/flb_downstream.c index 23e44691f08..6b5f4ac6124 100644 --- a/src/flb_downstream.c +++ b/src/flb_downstream.c @@ -107,6 +107,24 @@ static void flb_downstream_conn_event_coro(void) flb_downstream_conn_event_loop(connection, coro); } +/* Complete the setup of a socket that was accepted outside this coroutine */ +static int downstream_conn_adopted_setup(struct flb_connection *connection, + struct flb_coro *coro) +{ +#ifdef FLB_HAVE_TLS + if (flb_stream_is_secure(connection->stream) && + connection->stream->tls_context != NULL) { + return flb_tls_session_create(connection->stream->tls_context, + connection, coro); + } +#else + (void) connection; + (void) coro; +#endif + + return 0; +} + static void flb_downstream_conn_accept_event_coro(void) { int result; @@ -117,7 +135,13 @@ static void flb_downstream_conn_accept_event_coro(void) connection = coro->data; flb_connection_reset_connection_timeout(connection); - result = flb_io_net_accept(connection, coro); + if (connection->fd == FLB_INVALID_SOCKET) { + result = flb_io_net_accept(connection, coro); + } + else { + /* the socket was accepted by another worker and handed over */ + result = downstream_conn_adopted_setup(connection, coro); + } flb_connection_unset_connection_timeout(connection); if (result == 0 && @@ -721,8 +745,14 @@ int flb_downstream_conn_release_all(struct flb_downstream *stream) return 0; } -int flb_downstream_conn_event_accept( +/* + * Start the accept coroutine of a connection. With an invalid 'fd' the + * coroutine accepts from the listener, otherwise it adopts 'fd'. An adopted + * socket is owned by the connection once this function is entered. + */ +static int downstream_conn_event_start( struct flb_downstream *stream, + flb_sockfd_t fd, flb_connection_accept_callback accept_callback, void *accept_callback_data, flb_connection_event_callback event_callback, @@ -739,10 +769,18 @@ int flb_downstream_conn_event_accept( (mask & (MK_EVENT_READ | MK_EVENT_WRITE)) == 0 || (stream->base.transport != FLB_TRANSPORT_TCP && stream->base.transport != FLB_TRANSPORT_UNIX_STREAM)) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } return -1; } if (stream->paused == FLB_TRUE) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + return 0; + } + connection_fd = flb_net_accept(stream->server_fd); if (connection_fd >= 0) { flb_socket_close(connection_fd); @@ -755,20 +793,29 @@ int flb_downstream_conn_event_accept( config = stream->base.config; if (config == NULL || flb_downstream_is_async(stream) == FLB_FALSE) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } return -1; } - connection = flb_connection_create(FLB_INVALID_SOCKET, + connection = flb_connection_create(fd, FLB_DOWNSTREAM_CONNECTION, stream, flb_engine_evl_get(), NULL); if (connection == NULL) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } return -1; } coro = flb_coro_create(connection); if (coro == NULL) { + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } flb_connection_destroy(connection); return -1; } @@ -779,6 +826,9 @@ int flb_downstream_conn_event_accept( &stack_size); if (coro->callee == NULL) { flb_coro_destroy(coro); + if (fd != FLB_INVALID_SOCKET) { + flb_socket_close(fd); + } flb_connection_destroy(connection); return -1; } @@ -813,6 +863,35 @@ int flb_downstream_conn_event_accept( return ret; } +int flb_downstream_conn_event_accept( + struct flb_downstream *stream, + flb_connection_accept_callback accept_callback, + void *accept_callback_data, + flb_connection_event_callback event_callback, + int mask) +{ + return downstream_conn_event_start(stream, FLB_INVALID_SOCKET, + accept_callback, accept_callback_data, + event_callback, mask); +} + +int flb_downstream_conn_event_adopt( + struct flb_downstream *stream, + flb_sockfd_t fd, + flb_connection_accept_callback accept_callback, + void *accept_callback_data, + flb_connection_event_callback event_callback, + int mask) +{ + if (fd == FLB_INVALID_SOCKET) { + return -1; + } + + return downstream_conn_event_start(stream, fd, + accept_callback, accept_callback_data, + event_callback, mask); +} + int flb_downstream_conn_event_register(struct flb_connection *connection, flb_connection_event_callback callback, int mask) diff --git a/src/flb_input_ingest.c b/src/flb_input_ingest.c index 91ff0e0675b..0a5bd2a2cc3 100644 --- a/src/flb_input_ingest.c +++ b/src/flb_input_ingest.c @@ -52,6 +52,7 @@ struct flb_input_ingress_event { void *buf; size_t size; size_t records; + int processed; } log; struct ctrace *traces; struct cprof *profiles; @@ -378,7 +379,16 @@ static int flb_input_ingress_collector(struct flb_input_instance *ins, mk_list_entry_init(&event->_head); if (event->type == FLB_INPUT_INGRESS_LOG) { - if (event->data.log.records > 0) { + if (event->data.log.processed == FLB_TRUE) { + result = flb_input_log_append_processed_routed( + ins, + event->data.log.records, + event->tag, + event->tag != NULL ? flb_sds_len(event->tag) : 0, + event->data.log.buf, + event->data.log.size); + } + else if (event->data.log.records > 0) { result = flb_input_log_append_records( ins, event->data.log.records, @@ -571,13 +581,14 @@ int flb_input_ingress_queue_log_take(struct flb_input_instance *ins, allocation_size); } -int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, - size_t records, - const char *tag, - size_t tag_len, - void *buf, - size_t buf_size, - size_t allocation_size) +static int flb_input_ingress_queue_log_take_internal(struct flb_input_instance *ins, + size_t records, + int processed, + const char *tag, + size_t tag_len, + void *buf, + size_t buf_size, + size_t allocation_size) { struct flb_input_ingress_event *event; @@ -597,6 +608,7 @@ int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, event->data.log.buf = buf; event->data.log.size = buf_size; event->data.log.records = records; + event->data.log.processed = processed; if (allocation_size < buf_size) { allocation_size = buf_size; } @@ -605,6 +617,33 @@ int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, return flb_input_ingress_enqueue(ins, event); } +int flb_input_ingress_queue_log_take_records(struct flb_input_instance *ins, + size_t records, + const char *tag, + size_t tag_len, + void *buf, + size_t buf_size, + size_t allocation_size) +{ + return flb_input_ingress_queue_log_take_internal(ins, records, FLB_FALSE, + tag, tag_len, buf, buf_size, + allocation_size); +} + +int flb_input_ingress_queue_log_processed_take_records( + struct flb_input_instance *ins, + size_t records, + const char *tag, + size_t tag_len, + void *buf, + size_t buf_size, + size_t allocation_size) +{ + return flb_input_ingress_queue_log_take_internal(ins, records, FLB_TRUE, + tag, tag_len, buf, buf_size, + allocation_size); +} + int flb_input_ingress_queue_metrics(struct flb_input_instance *ins, const char *tag, size_t tag_len, diff --git a/src/flb_input_log.c b/src/flb_input_log.c index 764c9893fc1..255fea4f369 100644 --- a/src/flb_input_log.c +++ b/src/flb_input_log.c @@ -1526,6 +1526,44 @@ static int input_log_append_processed_internal(struct flb_input_instance *ins, return ret; } +/* Store a log buffer that already went through the input processors */ +static int input_log_append_output(struct flb_input_instance *ins, size_t records, + const char *tag, size_t tag_len, + const void *buf, size_t buf_size) +{ + if (flb_input_is_threaded(ins) == FLB_TRUE && + input_has_conditional_routes(ins) == FLB_TRUE) { + return flb_input_chunk_ring_buffer_enqueue_log_routing(ins, + FLB_INPUT_LOGS, + records, + tag, tag_len, + buf, buf_size); + } + + return input_log_append_processed_internal(ins, records, + tag, tag_len, + buf, buf_size, + FLB_FALSE); +} + +/* Tag used by processors when the input does not provide one */ +static void input_log_processor_tag(struct flb_input_instance *ins, + const char **tag, size_t *tag_len) +{ + if (*tag) { + return; + } + + if (ins->tag && ins->tag_len > 0) { + *tag = ins->tag; + *tag_len = ins->tag_len; + } + else { + *tag = ins->name; + *tag_len = strlen(ins->name); + } +} + /* * 'exact_records' tells whether 'records' is the exact number of log records in * the buffer. Caller-provided hints (for example, lines read by an input) are @@ -1546,16 +1584,7 @@ static int input_log_append(struct flb_input_instance *ins, processor_is_active = flb_processor_is_active(ins->processor); if (processor_is_active) { - if (!tag) { - if (ins->tag && ins->tag_len > 0) { - tag = ins->tag; - tag_len = ins->tag_len; - } - else { - tag = ins->name; - tag_len = strlen(ins->name); - } - } + input_log_processor_tag(ins, &tag, &tag_len); if (exact_records && records <= INT_MAX) { known_records = (int) records; @@ -1589,20 +1618,7 @@ static int input_log_append(struct flb_input_instance *ins, } } - if (flb_input_is_threaded(ins) == FLB_TRUE && - input_has_conditional_routes(ins) == FLB_TRUE) { - ret = flb_input_chunk_ring_buffer_enqueue_log_routing(ins, - FLB_INPUT_LOGS, - records, - tag, tag_len, - out_buf, out_size); - } - else { - ret = input_log_append_processed_internal(ins, records, - tag, tag_len, - out_buf, out_size, - FLB_FALSE); - } + ret = input_log_append_output(ins, records, tag, tag_len, out_buf, out_size); if (processor_is_active && buf != out_buf) { flb_free(out_buf); @@ -1641,6 +1657,25 @@ int flb_input_log_append_processed(struct flb_input_instance *ins, FLB_TRUE); } +/* + * Store an already processed buffer from a thread that might not own input + * chunks. Unlike flb_input_log_append_processed(), this entry point performs + * the normal threaded-input handoff instead of forcing a local append. + */ +int flb_input_log_append_processed_routed(struct flb_input_instance *ins, + size_t records, + const char *tag, + size_t tag_len, + const void *buf, + size_t buf_size) +{ + if (records == 0) { + records = flb_mp_count_log_records(buf, buf_size); + } + + return input_log_append_output(ins, records, tag, tag_len, buf, buf_size); +} + /* Take a msgpack serialized record and enqueue it as a chunk */ int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins, size_t processor_starting_stage, @@ -1670,3 +1705,55 @@ int flb_input_log_append_records(struct flb_input_instance *ins, ret = input_log_append(ins, 0, records, FLB_FALSE, tag, tag_len, buf, buf_size); return ret; } + +/* TRUE when the input processors can take log records built as a CFL chunk */ +int flb_input_log_chunk_supported(struct flb_input_instance *ins, + int worker_thread) +{ + return flb_processor_logs_chunk_supported(ins->processor, worker_thread); +} + +/* + * Run the input processors over a chunk of CFL records and enqueue the + * result, skipping the msgpack round trip the processors would otherwise do. + * The chunk is always consumed, callers check flb_input_log_chunk_supported() + * first. + */ +int flb_input_log_append_chunk(struct flb_input_instance *ins, + const char *tag, size_t tag_len, + struct flb_mp_chunk_cobj *chunk, + int worker_thread) +{ + int ret; + int records = -1; + void *out_buf = NULL; + size_t out_size = 0; + + input_log_processor_tag(ins, &tag, &tag_len); + + ret = flb_processor_run_logs_chunk(ins->processor, tag, tag_len, chunk, + worker_thread, + &out_buf, &out_size, &records); + if (ret == -1) { + return -1; + } + + if (out_size == 0) { + flb_free(out_buf); + return 0; + } + + if (records < 0) { + records = flb_mp_count_log_records(out_buf, out_size); + } + + if (ins->ingress_queue_enabled == FLB_TRUE) { + return flb_input_ingress_queue_log_processed_take_records( + ins, records, tag, tag_len, out_buf, out_size, out_size); + } + + ret = input_log_append_output(ins, records, tag, tag_len, out_buf, out_size); + flb_free(out_buf); + + return ret; +} diff --git a/src/flb_json.c b/src/flb_json.c index 3e8e1b5cb65..bf5bcc76d4f 100644 --- a/src/flb_json.c +++ b/src/flb_json.c @@ -18,6 +18,7 @@ */ #include +#include #include #include @@ -1877,3 +1878,39 @@ int flb_json_mut_obj_add_val(struct flb_json_mut_doc *document, return flb_json_mut_obj_add_val_len(document, object, key, strlen(key), value); } + +#ifdef FLB_HAVE_YYJSON +struct flb_json_tree *flb_json_tree_read(const char *input, size_t length) +{ + yyjson_alc allocator; + + if (input == NULL) { + return NULL; + } + + flb_json_yyjson_init_alc(&allocator); + + return (struct flb_json_tree *) yyjson_read_opts((char *) input, length, + YYJSON_READ_ALLOW_INVALID_UNICODE | + YYJSON_READ_REPLACE_INVALID_UNICODE, + &allocator, NULL); +} + +void flb_json_tree_destroy(struct flb_json_tree *tree) +{ + yyjson_doc_free((yyjson_doc *) tree); +} +#else +struct flb_json_tree *flb_json_tree_read(const char *input, size_t length) +{ + (void) input; + (void) length; + + return NULL; +} + +void flb_json_tree_destroy(struct flb_json_tree *tree) +{ + (void) tree; +} +#endif diff --git a/src/flb_mp.c b/src/flb_mp.c index d02e91281da..391aaa02cab 100644 --- a/src/flb_mp.c +++ b/src/flb_mp.c @@ -2008,7 +2008,11 @@ struct flb_mp_chunk_cobj *flb_mp_chunk_cobj_create(struct flb_log_event_encoder { struct flb_mp_chunk_cobj *chunk_cobj; - if (!log_encoder || !log_decoder) { + /* + * Both codecs can be NULL for a chunk whose records are built directly as + * CFL objects, the codecs are bound later by the processor that runs it. + */ + if ((log_encoder == NULL) != (log_decoder == NULL)) { return NULL; } @@ -2061,7 +2065,7 @@ int flb_mp_chunk_cobj_encode_append(struct flb_mp_chunk_cobj *chunk_cobj) msgpack_sbuffer buffer; msgpack_packer packer; - if (!chunk_cobj) { + if (!chunk_cobj || !chunk_cobj->log_encoder) { return -1; } @@ -2399,7 +2403,10 @@ int flb_mp_chunk_cobj_record_next(struct flb_mp_chunk_cobj *chunk_cobj, * populate the cobj list. Otherwise it means all the content is ready * as a chunk_cobj_record. */ - bytes = chunk_cobj->log_decoder->length - chunk_cobj->log_decoder->offset; + bytes = 0; + if (chunk_cobj->log_decoder) { + bytes = chunk_cobj->log_decoder->length - chunk_cobj->log_decoder->offset; + } if (bytes > 0) { record = flb_mp_chunk_record_create(chunk_cobj); diff --git a/src/flb_processor.c b/src/flb_processor.c index 8e68a2353a1..9e9cfaa55e4 100644 --- a/src/flb_processor.c +++ b/src/flb_processor.c @@ -1465,6 +1465,37 @@ static void chunk_codec_destroy(struct chunk_codec *codec) codec->ready = FLB_FALSE; } +#define CHUNK_MATERIALIZE_OK 0 +#define CHUNK_MATERIALIZE_PARTIAL 1 + +/* + * Decode every record while the per-run codec is private to this call. Keep + * the decoded prefix on an error, matching the former lazy callback behavior. + * record_next() does not distinguish malformed input from allocation/CFL + * conversion failures, so all three use prefix semantics here. + */ +static int chunk_cobj_materialize(struct flb_mp_chunk_cobj *chunk_cobj) +{ + int ret; + int records; + struct flb_mp_chunk_record *record; + + while (1) { + ret = flb_mp_chunk_cobj_record_next(chunk_cobj, &record); + if (ret == FLB_MP_CHUNK_RECORD_EOF) { + return CHUNK_MATERIALIZE_OK; + } + else if (ret != FLB_MP_CHUNK_RECORD_OK) { + records = flb_mp_chunk_cobj_count_log_records(chunk_cobj); + flb_warn("[processor] failed to decode record, processing %d decoded records", + records); + flb_log_event_decoder_reset(chunk_cobj->log_decoder, NULL, 0); + chunk_cobj->record_pos = NULL; + return CHUNK_MATERIALIZE_PARTIAL; + } + } +} + /* * Large log buffers, such as complete chunks reaching output processors, are * processed in windows when every native unit of the run is record local: @@ -1535,9 +1566,9 @@ static struct log_window *log_windows_scan(void *data, size_t data_size, int *co FLB_PROCESSOR_LOG_WINDOW_BYTES, FLB_PROCESSOR_LOG_WINDOW_RECORDS, &end, &records) != 0) { - /* the single chunk path reports invalid content as usual */ - flb_free(windows); - return NULL; + /* Let the final window preserve records before a corrupt tail. */ + end = data_size; + records = 0; } if (*count == capacity) { @@ -1657,7 +1688,12 @@ static int run_log_windows(struct flb_processor *proc, struct chunk_codec *codec goto error; } - records = windows[w].records; + ret = chunk_cobj_materialize(chunk_cobj); + if (ret == CHUNK_MATERIALIZE_PARTIAL) { + units[0].error = FLB_TRUE; + } + + records = flb_mp_chunk_cobj_count_log_records(chunk_cobj); for (i = 0; i < unit_count; i++) { p_ins = units[i].pu->ctx; @@ -1777,6 +1813,7 @@ int flb_processor_run(struct flb_processor *proc, * or a CMetrics, CTraces or CProfiles context for the other signal types. */ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, + struct flb_mp_chunk_cobj *input_chunk, size_t starting_stage, int type, const char *tag, size_t tag_len, @@ -1791,6 +1828,7 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, int item_metrics_available; int finalize; int raw_result; + int unit_lock_held; int log_buffer_normalized = FLB_FALSE; int log_buffer_records = input_records; void *cur_buf = NULL; @@ -1851,6 +1889,18 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, cur_buf = data; cur_size = data_size; + /* a chunk built by the caller replaces the msgpack decoding step */ + if (input_chunk) { + if (chunk_codec_prepare(codec) != 0) { + flb_mp_chunk_cobj_destroy(input_chunk); + return -1; + } + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); + input_chunk->log_encoder = &codec->encoder; + input_chunk->log_decoder = &codec->decoder; + chunk_cobj = input_chunk; + } + /* iterate list units */ mk_list_foreach(head, list) { pu = mk_list_entry(head, struct flb_processor_unit, _head); @@ -1866,6 +1916,7 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, tmp_buf = NULL; tmp_size = 0; pu_error = FLB_FALSE; + unit_lock_held = FLB_FALSE; unit_in_items = -1; unit_out_items = -1; item_metrics_available = FLB_FALSE; @@ -1953,13 +2004,52 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, } } + /* + * The decoder and CFL chunk belong to this processor_run() call, not + * to a processor instance. Materialize the complete input before + * taking the unit lock so concurrent calls do not serialize codec + * work behind an otherwise record-local callback. + */ + if (type == FLB_PROCESSOR_LOGS && chunk_cobj == NULL && + pu->unit_type == FLB_PROCESSOR_UNIT_NATIVE) { + p_ins = pu->ctx; + + if (p_ins->p->cb_process_logs != NULL) { + if (chunk_codec_prepare(codec) != 0) { + if (cur_buf != data) { + flb_free(cur_buf); + } + return -1; + } + + flb_log_event_decoder_reset(&codec->decoder, cur_buf, cur_size); + chunk_cobj = flb_mp_chunk_cobj_create(&codec->encoder, &codec->decoder); + if (chunk_cobj == NULL) { + flb_log_event_decoder_reset(&codec->decoder, NULL, 0); + if (cur_buf != data) { + flb_free(cur_buf); + } + return -1; + } + + ret = chunk_cobj_materialize(chunk_cobj); + if (ret == CHUNK_MATERIALIZE_PARTIAL) { + pu_error = FLB_TRUE; + } + } + } + ret = acquire_lock(&pu->lock, FLB_PROCESSOR_LOCK_RETRY_LIMIT, FLB_PROCESSOR_LOCK_RETRY_DELAY); if (ret != FLB_TRUE) { + if (chunk_cobj) { + flb_mp_chunk_cobj_destroy(chunk_cobj); + } return -1; } + unit_lock_held = FLB_TRUE; /* run the unit */ if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { @@ -2102,55 +2192,6 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, if (type == FLB_PROCESSOR_LOGS) { if (p_ins->p->cb_process_logs != NULL) { - /* if no previous chunkj_cobj exist, create instance. Note that this context will last - * until no more processors exists or the next one is a "filter" type processor. - */ - if (!chunk_cobj) { - if (chunk_codec_prepare(codec) != 0) { - if (cur_buf != data) { - flb_free(cur_buf); - } - pu_error = FLB_TRUE; - processor_metrics_update(proc, pu, - metrics_scope, - metrics_owner, - metrics_signal, - unit_in_items, - unit_out_items, - item_metrics_available, - pu_error); - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - return -1; - } - flb_log_event_decoder_reset(&codec->decoder, cur_buf, cur_size); - - /* create the context */ - chunk_cobj = flb_mp_chunk_cobj_create(&codec->encoder, &codec->decoder); - if (chunk_cobj == NULL) { - flb_log_event_decoder_reset(&codec->decoder, NULL, 0); - if (cur_buf != data) { - flb_free(cur_buf); - } - - pu_error = FLB_TRUE; - processor_metrics_update(proc, pu, - metrics_scope, - metrics_owner, - metrics_signal, - unit_in_items, - unit_out_items, - item_metrics_available, - pu_error); - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - - return -1; - } - } - /* Set the processor condition if it exists */ if (pu->condition) { flb_debug("[processor] setting condition for chunk processing (pu=%s)", pu->name); @@ -2170,6 +2211,10 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, /* Clear the condition after processing */ chunk_cobj->condition = NULL; + release_lock(&pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + unit_lock_held = FLB_FALSE; finalize = FLB_FALSE; /* is this processing_unit the last one from the list ? */ @@ -2219,9 +2264,6 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, unit_out_items, item_metrics_available, pu_error); - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); return 0; } @@ -2236,10 +2278,6 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, flb_free(cur_buf); } - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); - pu_error = FLB_TRUE; processor_metrics_update(proc, pu, metrics_scope, @@ -2410,9 +2448,11 @@ static int processor_run(struct flb_processor *proc, struct chunk_codec *codec, unit_out_items, item_metrics_available, pu_error); - release_lock(&pu->lock, - FLB_PROCESSOR_LOCK_RETRY_LIMIT, - FLB_PROCESSOR_LOCK_RETRY_DELAY); + if (unit_lock_held == FLB_TRUE) { + release_lock(&pu->lock, + FLB_PROCESSOR_LOCK_RETRY_LIMIT, + FLB_PROCESSOR_LOCK_RETRY_DELAY); + } } /* set output buffer */ @@ -2485,7 +2525,7 @@ int flb_processor_run_with_count(struct flb_processor *proc, struct chunk_codec codec; codec.ready = FLB_FALSE; - ret = processor_run(proc, &codec, starting_stage, type, tag, tag_len, + ret = processor_run(proc, &codec, NULL, starting_stage, type, tag, tag_len, data, data_size, out_buf, out_size, input_records, output_records); chunk_codec_destroy(&codec); @@ -2493,6 +2533,82 @@ int flb_processor_run_with_count(struct flb_processor *proc, return ret; } +/* + * A logs chunk built directly as CFL records can enter the pipeline when the + * every unit is a native processor that processes logs. A worker thread also + * requires every unit to be worker-safe. The whole chain is checked before + * any unit runs so fallback can never execute an already-processed prefix a + * second time. Filters are excluded even in non-worker callers because mixed + * native/filter chains cross a serialization boundary between units. + */ +int flb_processor_logs_chunk_supported(struct flb_processor *proc, + int worker_thread) +{ + struct mk_list *head; + struct flb_processor_unit *pu; + struct flb_processor_instance *p_ins; + + if (!proc || !proc->is_active || mk_list_is_empty(&proc->logs) == 0) { + return FLB_FALSE; + } + + mk_list_foreach(head, &proc->logs) { + pu = mk_list_entry(head, struct flb_processor_unit, _head); + if (pu->unit_type == FLB_PROCESSOR_UNIT_FILTER) { + return FLB_FALSE; + } + + p_ins = pu->ctx; + if (p_ins->p->cb_process_logs == NULL) { + return FLB_FALSE; + } + if (worker_thread == FLB_TRUE && + (p_ins->p->flags & FLB_PROCESSOR_FLAG_LOGS_WORKER_SAFE) == 0) { + return FLB_FALSE; + } + } + + return FLB_TRUE; +} + +/* + * Run the logs processors over a chunk of CFL records. The chunk is always + * consumed. On success the output buffer is owned by the caller, it is NULL + * with a zero size when every record was dropped. + */ +int flb_processor_run_logs_chunk(struct flb_processor *proc, + const char *tag, size_t tag_len, + struct flb_mp_chunk_cobj *chunk, + int worker_thread, + void **out_buf, size_t *out_size, + int *output_records) +{ + int ret; + struct chunk_codec codec; + + *out_buf = NULL; + *out_size = 0; + if (output_records) { + *output_records = -1; + } + + if (!chunk) { + return -1; + } + + if (flb_processor_logs_chunk_supported(proc, worker_thread) == FLB_FALSE) { + flb_mp_chunk_cobj_destroy(chunk); + return -1; + } + + codec.ready = FLB_FALSE; + ret = processor_run(proc, &codec, chunk, 0, FLB_PROCESSOR_LOGS, tag, tag_len, + NULL, 0, out_buf, out_size, -1, output_records); + chunk_codec_destroy(&codec); + + return ret; +} + void flb_processor_destroy(struct flb_processor *proc) { struct mk_list *head; diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index 19b83452ef4..eba86b7a5f1 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -41,6 +42,32 @@ struct flb_http_server_worker_context { struct flb_net_setup net_setup; }; +/* + * Connection balancing between managed workers. Every worker accepts from + * its own SO_REUSEPORT listener and the kernel spreads connections over + * those listeners by hash, so a few long-lived connections can pile up on one + * worker while the others stay idle. The accepting worker therefore hands + * each new socket to the worker that owns the fewest connections. + */ +struct flb_http_server_handoff { + struct mk_event event; /* must be the first member */ + pthread_mutex_t lock; + uint64_t ready; + flb_pipefd_t channel[2]; + flb_sockfd_t *queue; + size_t queue_length; + size_t queue_size; + uint64_t connections; /* sessions owned by the worker */ + uint64_t pending; /* sockets waiting in the queue */ + struct flb_http_server *server; +}; + +struct flb_http_server_balancer { + int workers; + uint64_t next; + struct flb_http_server_handoff *slots; +}; + static void flb_http_server_runtime_stop(struct flb_http_server *session); static void flb_http_server_pause_on_event_loop(struct flb_http_server *server); @@ -116,6 +143,23 @@ static void flb_http_server_reap_stale_sessions(struct flb_http_server *server) } } +static void flb_http_server_counter_add(uint64_t *counter, int64_t delta) +{ + uint64_t value; + + while (FLB_TRUE) { + value = cfl_atomic_load(counter); + + if (delta < 0 && value < (uint64_t) -delta) { + return; + } + + if (cfl_atomic_compare_exchange(counter, value, value + delta)) { + return; + } + } +} + static int flb_http_server_connection_slot_reserve(struct flb_http_server *server) { uint64_t connection_count; @@ -204,6 +248,8 @@ static int flb_http_server_apply_options(struct flb_http_server *session, session->cb_worker_init = options->cb_worker_init; session->cb_worker_exit = options->cb_worker_exit; session->runtime = NULL; + session->balancer = NULL; + session->handoff = NULL; cfl_list_init(&session->clients); cfl_list_init(&session->detached_clients); @@ -566,6 +612,11 @@ static int flb_http_server_client_connection_initialize( session->connection = connection; session->connection_slot_reserved = FLB_TRUE; + if (server->handoff != NULL) { + flb_http_server_counter_add(&server->handoff->connections, 1); + session->balanced_connection = FLB_TRUE; + } + if (session->version <= HTTP_PROTOCOL_VERSION_11) { session->http1.stream.user_data = server->user_data; } @@ -591,6 +642,172 @@ static int flb_http_server_client_connection_initialize( return 0; } +static int flb_http_server_connection_adopt(struct flb_http_server *server, + flb_sockfd_t fd) +{ + return flb_downstream_conn_event_adopt(server->downstream, fd, + flb_http_server_client_connection_initialize, + server, + flb_http_server_client_activity_event_handler, + MK_EVENT_READ); +} + +/* Worker owning the fewest connections, rotating the start to break ties */ +static struct flb_http_server_handoff *flb_http_server_balancer_pick( + struct flb_http_server_balancer *balancer) +{ + int i; + int index; + int start; + uint64_t load; + uint64_t best_load; + struct flb_http_server_handoff *slot; + struct flb_http_server_handoff *best; + + best = NULL; + best_load = 0; + start = (int) (cfl_atomic_load(&balancer->next) % balancer->workers); + flb_http_server_counter_add(&balancer->next, 1); + + for (i = 0; i < balancer->workers; i++) { + index = (start + i) % balancer->workers; + slot = &balancer->slots[index]; + + if (cfl_atomic_load(&slot->ready) == FLB_FALSE) { + continue; + } + + load = cfl_atomic_load(&slot->connections) + cfl_atomic_load(&slot->pending); + if (best == NULL || load < best_load) { + best = slot; + best_load = load; + } + } + + return best; +} + +/* Queue a socket for another worker, the socket is not consumed on failure */ +static int flb_http_server_handoff_push(struct flb_http_server_handoff *slot, + flb_sockfd_t fd) +{ + char signal; + size_t size; + flb_sockfd_t *queue; + + pthread_mutex_lock(&slot->lock); + + if (slot->ready == FLB_FALSE) { + pthread_mutex_unlock(&slot->lock); + return -1; + } + + if (slot->queue_length == slot->queue_size) { + size = slot->queue_size == 0 ? 16 : slot->queue_size * 2; + queue = flb_realloc(slot->queue, size * sizeof(flb_sockfd_t)); + if (queue == NULL) { + flb_errno(); + pthread_mutex_unlock(&slot->lock); + return -1; + } + slot->queue = queue; + slot->queue_size = size; + } + + /* one notification per empty to non-empty transition */ + if (slot->queue_length == 0) { + signal = 1; + if (flb_pipe_w(slot->channel[1], &signal, sizeof(signal)) != sizeof(signal)) { + pthread_mutex_unlock(&slot->lock); + return -1; + } + } + + slot->queue[slot->queue_length++] = fd; + flb_http_server_counter_add(&slot->pending, 1); + + pthread_mutex_unlock(&slot->lock); + + return 0; +} + +/* Runs on the worker owning 'slot' when sockets were queued for it */ +static int flb_http_server_handoff_event_handler(void *data) +{ + char signal; + size_t i; + size_t length; + flb_sockfd_t *queue; + struct flb_http_server_handoff *slot; + + slot = (struct flb_http_server_handoff *) data; + + if (flb_pipe_r(slot->channel[0], &signal, sizeof(signal)) != sizeof(signal)) { + return -1; + } + + pthread_mutex_lock(&slot->lock); + queue = slot->queue; + length = slot->queue_length; + slot->queue = NULL; + slot->queue_length = 0; + slot->queue_size = 0; + flb_http_server_counter_add(&slot->pending, -((int64_t) length)); + pthread_mutex_unlock(&slot->lock); + + for (i = 0; i < length; i++) { + /* the session reserves the slot again while it is set up */ + flb_http_server_connection_slot_release(slot->server); + flb_http_server_connection_adopt(slot->server, queue[i]); + } + + if (queue != NULL) { + flb_free(queue); + } + + return 0; +} + +/* Accept from this worker listener and give each socket to the least loaded worker */ +static int flb_http_server_balanced_accept(struct flb_http_server *server) +{ + int accepted; + flb_sockfd_t fd; + struct flb_http_server_handoff *target; + + for (accepted = 0; accepted < FLB_HTTP_SERVER_ACCEPT_BATCH_SIZE; accepted++) { + fd = flb_net_accept(server->downstream->server_fd); + if (fd == FLB_INVALID_SOCKET) { + break; + } + + if (server->downstream->paused == FLB_TRUE) { + flb_socket_close(fd); + continue; + } + + target = flb_http_server_balancer_pick(server->balancer); + + /* + * A connection limit slot is held while the socket waits for the + * other worker, so connections are still admitted in accept order. + * When no slot is free the socket stays here and the session setup + * rejects it as usual. + */ + if (target != NULL && target != server->handoff && + flb_http_server_connection_slot_reserve(server)) { + if (flb_http_server_handoff_push(target, fd) == 0) { + continue; + } + flb_http_server_connection_slot_release(server); + } + + flb_http_server_connection_adopt(server, fd); + } + + return 0; +} + static int flb_http_server_client_connection_event_handler(void *data) { int accepted_connections; @@ -626,6 +843,10 @@ static int flb_http_server_client_connection_event_handler(void *data) return result; } + if (server->balancer != NULL && server->handoff != NULL) { + return flb_http_server_balanced_accept(server); + } + accepted_connections = 0; do { @@ -664,6 +885,129 @@ static void flb_http_server_worker_maintenance(struct flb_downstream_worker *wor flb_http_server_reap_stale_sessions(&context->server); } +static void flb_http_server_balancer_destroy(struct flb_http_server_balancer *balancer) +{ + int i; + + if (balancer == NULL) { + return; + } + + for (i = 0; i < balancer->workers; i++) { + pthread_mutex_destroy(&balancer->slots[i].lock); + } + + flb_free(balancer->slots); + flb_free(balancer); +} + +static struct flb_http_server_balancer *flb_http_server_balancer_create(int workers) +{ + int i; + struct flb_http_server_balancer *balancer; + + balancer = flb_calloc(1, sizeof(struct flb_http_server_balancer)); + if (balancer == NULL) { + flb_errno(); + return NULL; + } + + balancer->slots = flb_calloc(workers, sizeof(struct flb_http_server_handoff)); + if (balancer->slots == NULL) { + flb_errno(); + flb_free(balancer); + return NULL; + } + + for (i = 0; i < workers; i++) { + if (pthread_mutex_init(&balancer->slots[i].lock, NULL) != 0) { + balancer->workers = i; + flb_http_server_balancer_destroy(balancer); + return NULL; + } + balancer->slots[i].channel[0] = -1; + balancer->slots[i].channel[1] = -1; + } + balancer->workers = workers; + + return balancer; +} + +/* Register the handoff channel of a worker, runs on the worker thread */ +static int flb_http_server_handoff_attach(struct flb_http_server *server, + struct flb_http_server_balancer *balancer, + int worker_id) +{ + int result; + struct flb_http_server_handoff *slot; + + if (worker_id < 0 || worker_id >= balancer->workers) { + return -1; + } + + slot = &balancer->slots[worker_id]; + + result = flb_pipe_create(slot->channel); + if (result != 0) { + return -1; + } + + MK_EVENT_NEW(&slot->event); + slot->event.type = FLB_ENGINE_EV_CUSTOM; + slot->event.handler = flb_http_server_handoff_event_handler; + + result = mk_event_add(server->event_loop, slot->channel[0], + FLB_ENGINE_EV_CUSTOM, MK_EVENT_READ, &slot->event); + if (result == -1) { + flb_pipe_destroy(slot->channel); + slot->channel[0] = -1; + slot->channel[1] = -1; + return -1; + } + + slot->server = server; + server->balancer = balancer; + server->handoff = slot; + + pthread_mutex_lock(&slot->lock); + cfl_atomic_store(&slot->ready, FLB_TRUE); + pthread_mutex_unlock(&slot->lock); + + return 0; +} + +/* Stop receiving sockets and close the ones still queued, runs on the worker thread */ +static void flb_http_server_handoff_detach(struct flb_http_server *server) +{ + size_t i; + struct flb_http_server_handoff *slot; + + slot = server->handoff; + if (slot == NULL) { + return; + } + + pthread_mutex_lock(&slot->lock); + cfl_atomic_store(&slot->ready, FLB_FALSE); + for (i = 0; i < slot->queue_length; i++) { + flb_http_server_connection_slot_release(server); + flb_socket_close(slot->queue[i]); + } + if (slot->queue != NULL) { + flb_free(slot->queue); + } + slot->queue = NULL; + slot->queue_length = 0; + slot->queue_size = 0; + cfl_atomic_store(&slot->pending, 0); + pthread_mutex_unlock(&slot->lock); + + mk_event_del(server->event_loop, &slot->event); + flb_pipe_destroy(slot->channel); + slot->channel[0] = -1; + slot->channel[1] = -1; +} + static int flb_http_server_worker_initialize(struct flb_downstream_worker *worker, struct flb_http_server *parent, struct flb_http_server_worker_context *context) @@ -718,6 +1062,15 @@ static int flb_http_server_worker_initialize(struct flb_downstream_worker *worke context->server.worker_id = flb_downstream_worker_id_get(worker); context->server.workers = parent->workers; + if (parent->balancer != NULL) { + result = flb_http_server_handoff_attach(&context->server, parent->balancer, + context->server.worker_id); + if (result != 0) { + flb_http_server_destroy(&context->server); + return result; + } + } + return 0; } @@ -746,6 +1099,7 @@ static int flb_http_server_worker_init(struct flb_downstream_worker *worker, ret = flb_downstream_worker_listener_fd_set( worker, context->server.downstream->server_fd); if (ret != 0) { + flb_http_server_handoff_detach(&context->server); flb_http_server_destroy(&context->server); flb_free(context); return ret; @@ -765,6 +1119,7 @@ static void flb_http_server_worker_exit(struct flb_downstream_worker *worker, context = worker_context; + flb_http_server_handoff_detach(&context->server); flb_http_server_destroy(&context->server); flb_free(context); } @@ -787,6 +1142,13 @@ static int flb_http_server_runtime_start(struct flb_http_server *session) session->tls_alpn_configured = FLB_TRUE; } + if (session->workers > 1) { + session->balancer = flb_http_server_balancer_create(session->workers); + if (session->balancer == NULL) { + return -1; + } + } + memset(&options, 0, sizeof(struct flb_downstream_worker_options)); options.workers = session->workers; options.parent = session; @@ -796,6 +1158,8 @@ static int flb_http_server_runtime_start(struct flb_http_server *session) result = flb_downstream_worker_runtime_start(&session->runtime, &options); if (result != 0) { + flb_http_server_balancer_destroy(session->balancer); + session->balancer = NULL; return -1; } @@ -808,6 +1172,9 @@ static void flb_http_server_runtime_stop(struct flb_http_server *session) { flb_downstream_worker_runtime_stop(session->runtime); session->runtime = NULL; + + flb_http_server_balancer_destroy(session->balancer); + session->balancer = NULL; } /* HTTP SERVER */ @@ -1358,6 +1725,11 @@ void flb_http_server_session_destroy(struct flb_http_server_session *session) session->connection_slot_reserved = FLB_FALSE; } + if (session->balanced_connection && session->parent->handoff != NULL) { + flb_http_server_counter_add(&session->parent->handoff->connections, -1); + session->balanced_connection = FLB_FALSE; + } + if (session->incoming_data != NULL) { cfl_sds_destroy(session->incoming_data); } diff --git a/src/opentelemetry/flb_opentelemetry_logs_json.c b/src/opentelemetry/flb_opentelemetry_logs_json.c new file mode 100644 index 00000000000..675053507ff --- /dev/null +++ b/src/opentelemetry/flb_opentelemetry_logs_json.c @@ -0,0 +1,1509 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#define OTEL_JSON_CFL_ARENA_CHUNK_SIZE 16384 +/* + * The established path builds maps and arrays with 32-bit headers that are + * patched once the entries are known, the logs_max_size accounting counts + * them the same way. Values it copies unchanged keep compact headers. + */ +#define OTEL_JSON_CFL_MAP_HEADER_SIZE 5 +#define OTEL_JSON_CFL_ARRAY_HEADER_SIZE 5 +#define OTEL_JSON_CFL_RECORD_SIZE 12 +#define OTEL_JSON_CFL_MAX_DEPTH 32 + +struct json_cfl_builder { + const struct flb_opentelemetry_logs_decode_options *options; + struct flb_mp_chunk_cobj *chunk; + struct cfl_arena *arena; + size_t size; + size_t limit; + int error_status; + int unsupported; +}; + +struct flb_opentelemetry_logs_json_cfl { + struct flb_mp_chunk_cobj *chunk; + struct cfl_arena *arena; + struct flb_json_tree *document; + size_t encoded_size; + size_t record_count; +}; + + +static void *json_arena_malloc(void *context, size_t size) +{ + (void) context; + return flb_malloc(size); +} + +static void json_arena_free(void *context, void *pointer) +{ + (void) context; + flb_free(pointer); +} + +static struct cfl_arena *json_arena_create(void) +{ + struct cfl_arena_options options; + + cfl_arena_options_init(&options); + options.chunk_size = OTEL_JSON_CFL_ARENA_CHUNK_SIZE; + options.maximum_chunk_size = 65536; + options.malloc_fn = json_arena_malloc; + options.free_fn = json_arena_free; + + return cfl_arena_create_with_options(&options); +} + +static int json_cfl_account(struct json_cfl_builder *builder, size_t bytes) +{ + if (bytes > builder->limit || builder->size > builder->limit - bytes) { + return -1; + } + builder->size += bytes; + return 0; +} + +static size_t json_cfl_uint_size(uint64_t value) +{ + if (value <= 0x7f) { + return 1; + } + if (value <= UINT8_MAX) { + return 2; + } + if (value <= UINT16_MAX) { + return 3; + } + if (value <= UINT32_MAX) { + return 5; + } + return 9; +} + +static size_t json_cfl_int_size(int64_t value) +{ + if (value >= -32) { + return 1; + } + if (value >= INT8_MIN) { + return 2; + } + if (value >= INT16_MIN) { + return 3; + } + if (value >= INT32_MIN) { + return 5; + } + return 9; +} + +static size_t json_cfl_string_header_size(size_t length) +{ + if (length <= 31) { + return 1; + } + if (length <= UINT8_MAX) { + return 2; + } + if (length <= UINT16_MAX) { + return 3; + } + return 5; +} + +static size_t json_cfl_binary_header_size(size_t length) +{ + if (length <= UINT8_MAX) { + return 2; + } + if (length <= UINT16_MAX) { + return 3; + } + return 5; +} + +static size_t json_cfl_array_header_size(size_t entries) +{ + if (entries <= 15) { + return 1; + } + if (entries <= UINT16_MAX) { + return 3; + } + return 5; +} + +static int json_cfl_account_sized(struct json_cfl_builder *builder, + size_t header, size_t value) +{ + if (json_cfl_account(builder, header) != 0) { + return -1; + } + return json_cfl_account(builder, value); +} + +static struct cfl_variant *json_cfl_string(struct json_cfl_builder *builder, + const char *value, size_t length) +{ + if (json_cfl_account_sized(builder, json_cfl_string_header_size(length), length) != 0) { + return NULL; + } + return cfl_variant_create_from_string_s_in(builder->arena, + (char *) value, length, CFL_TRUE); +} + +static struct cfl_variant *json_cfl_uint(struct json_cfl_builder *builder, + uint64_t value) +{ + if (json_cfl_account(builder, json_cfl_uint_size(value)) != 0) { + return NULL; + } + return cfl_variant_create_from_uint64_in(builder->arena, value); +} + +static struct cfl_variant *json_cfl_int(struct json_cfl_builder *builder, + int64_t value) +{ + if (value >= 0) { + return json_cfl_uint(builder, (uint64_t) value); + } + if (json_cfl_account(builder, json_cfl_int_size(value)) != 0) { + return NULL; + } + return cfl_variant_create_from_int64_in(builder->arena, value); +} + +static struct cfl_variant *json_cfl_double(struct json_cfl_builder *builder, + double value) +{ + if (json_cfl_account(builder, 9) != 0) { + return NULL; + } + return cfl_variant_create_from_double_in(builder->arena, value); +} + +static struct cfl_variant *json_cfl_bool(struct json_cfl_builder *builder, int value) +{ + if (json_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_bool_in(builder->arena, value); +} + +static struct cfl_variant *json_cfl_null(struct json_cfl_builder *builder) +{ + if (json_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_null_in(builder->arena); +} + +static struct cfl_kvlist *json_cfl_kvlist_create(struct json_cfl_builder *builder) +{ + if (json_cfl_account(builder, OTEL_JSON_CFL_MAP_HEADER_SIZE) != 0) { + return NULL; + } + return cfl_kvlist_create_in(builder->arena); +} + +static int json_cfl_insert(struct json_cfl_builder *builder, + struct cfl_kvlist *kvlist, + const char *key, size_t key_length, + struct cfl_variant *value) +{ + if (value == NULL) { + return -1; + } + if (json_cfl_account_sized(builder, json_cfl_string_header_size(key_length), + key_length) != 0 || + cfl_kvlist_insert_s(kvlist, (char *) key, key_length, value) != 0) { + cfl_variant_destroy(value); + return -1; + } + return 0; +} + +static int json_cfl_insert_kvlist(struct json_cfl_builder *builder, + struct cfl_kvlist *target, + const char *key, size_t key_length, + struct cfl_kvlist *value) +{ + struct cfl_variant *variant; + + if (value == NULL) { + return -1; + } + variant = cfl_variant_create_from_kvlist_in(builder->arena, value); + if (variant == NULL) { + cfl_kvlist_destroy(value); + return -1; + } + return json_cfl_insert(builder, target, key, key_length, variant); +} + +static struct cfl_object *json_cfl_object(struct json_cfl_builder *builder, + struct cfl_kvlist *kvlist) +{ + struct cfl_object *object; + + if (kvlist == NULL) { + return NULL; + } + object = cfl_object_create(); + if (object == NULL) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + object->variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); + if (object->variant == NULL) { + cfl_kvlist_destroy(kvlist); + cfl_object_destroy(object); + return NULL; + } + object->type = CFL_OBJECT_KVLIST; + return object; +} + +static struct flb_mp_chunk_record *json_cfl_record_add(struct json_cfl_builder *builder, + struct flb_time *timestamp) +{ + struct flb_mp_chunk_record *record; + + if (json_cfl_account(builder, OTEL_JSON_CFL_RECORD_SIZE) != 0) { + return NULL; + } + record = flb_mp_chunk_record_create(builder->chunk); + if (record == NULL) { + return NULL; + } + memset(&record->event, 0, sizeof(record->event)); + flb_time_copy(&record->event.timestamp, timestamp); + cfl_list_add(&record->_head, &builder->chunk->records); + return record; +} + +/* + * Keys are compared case-insensitively. The expected length is known at + * compile time, so most keys are rejected by their length alone; callers + * pass string literals through the json_obj_get() macro. + */ +#define json_obj_get(object, literal) \ + flb_json_node_obj_get_case(object, "" literal, sizeof(literal) - 1) + +static int json_base64_decode(const char *source, size_t source_size, + unsigned char **output, size_t *output_size) +{ + int result; + char value; + char *normalized; + size_t index; + size_t padded_size; + unsigned char *decoded; + + *output = NULL; + *output_size = 0; + if (source_size % 4 == 1) { + return -1; + } + padded_size = source_size + ((4 - source_size % 4) % 4); + normalized = flb_malloc(padded_size + 1); + decoded = flb_malloc((padded_size / 4) * 3 + 1); + if (normalized == NULL || decoded == NULL) { + flb_free(normalized); + flb_free(decoded); + return -1; + } + for (index = 0; index < source_size; index++) { + value = source[index]; + normalized[index] = value == '-' ? '+' : (value == '_' ? '/' : value); + } + for (; index < padded_size; index++) { + normalized[index] = '='; + } + normalized[padded_size] = '\0'; + result = flb_base64_decode(decoded, (padded_size / 4) * 3 + 1, output_size, + (unsigned char *) normalized, padded_size); + flb_free(normalized); + if (result != 0) { + flb_free(decoded); + return -1; + } + *output = decoded; + return 0; +} + +static struct cfl_variant *json_plain_value(struct json_cfl_builder *builder, + struct flb_json_node *value); +static struct cfl_variant *json_converted_value(struct json_cfl_builder *builder, + struct flb_json_node *value, size_t depth); + +/* + * The conversion of OTLP JSON values below mirrors the established msgpack + * path in flb_opentelemetry_utils.c function by function, including its + * leniency, so both paths produce the same records: + * + * json_wrapped() flb_otel_utils_json_payload_get_wrapped_value() + * json_unwrapped_value() flb_otel_utils_json_payload_append_unwrapped_value() + * json_converted_*() flb_otel_utils_json_payload_append_converted_*() + * + * A result CFL cannot hold, a map key that is not a string, marks the + * payload unsupported so the caller can use the msgpack path instead. + */ +#define JSON_WRAPPED_STRING 1 +#define JSON_WRAPPED_BOOL 2 +#define JSON_WRAPPED_INT 3 +#define JSON_WRAPPED_DOUBLE 4 +#define JSON_WRAPPED_BYTES 5 +#define JSON_WRAPPED_ARRAY 6 +#define JSON_WRAPPED_KVLIST 7 + +/* like the established path, a wrapper key matches any prefix of the name */ +static int json_wrapper_key(const char *key, size_t key_length, const char *name) +{ + return strncasecmp(key, name, key_length) == 0; +} + +static int json_wrapped(struct flb_json_node *object, struct flb_json_node **out_value, + int *out_type) +{ + int type; + int value_type; + size_t key_length; + const char *key_name; + struct flb_json_node *key; + struct flb_json_node *value; + + if (flb_json_node_type(object) != FLB_JSON_NODE_OBJ) { + return -1; + } + type = -1; + value = NULL; + value_type = FLB_JSON_NODE_NONE; + if (flb_json_node_obj_size(object) == 1) { + key = flb_json_node_first(object); + value = flb_json_node_key_value(key); + key_name = flb_json_node_get_strn(key, &key_length); + value_type = flb_json_node_type(value); + + if (json_wrapper_key(key_name, key_length, "stringValue")) { + if (value_type != FLB_JSON_NODE_NULL && value_type != FLB_JSON_NODE_STR) { + return -2; + } + type = JSON_WRAPPED_STRING; + } + else if (json_wrapper_key(key_name, key_length, "boolValue")) { + if (value_type != FLB_JSON_NODE_BOOL) { + return -2; + } + type = JSON_WRAPPED_BOOL; + } + else if (json_wrapper_key(key_name, key_length, "intValue")) { + if (value_type != FLB_JSON_NODE_UINT && value_type != FLB_JSON_NODE_SINT && + value_type != FLB_JSON_NODE_STR) { + return -2; + } + type = JSON_WRAPPED_INT; + } + else if (json_wrapper_key(key_name, key_length, "doubleValue")) { + if (value_type != FLB_JSON_NODE_REAL && value_type != FLB_JSON_NODE_STR) { + return -2; + } + type = JSON_WRAPPED_DOUBLE; + } + else if (json_wrapper_key(key_name, key_length, "bytesValue")) { + if (value_type != FLB_JSON_NODE_STR) { + return -2; + } + type = JSON_WRAPPED_BYTES; + } + else if (json_wrapper_key(key_name, key_length, "arrayValue")) { + if (value_type != FLB_JSON_NODE_ARR && value_type != FLB_JSON_NODE_OBJ) { + return -2; + } + type = JSON_WRAPPED_ARRAY; + } + else if (json_wrapper_key(key_name, key_length, "kvlistValue")) { + if (value_type != FLB_JSON_NODE_OBJ) { + return -2; + } + type = JSON_WRAPPED_KVLIST; + } + } + if (type == -1) { + return -1; + } + if (out_type != NULL) { + *out_type = type; + } + if (value_type == FLB_JSON_NODE_OBJ && flb_json_node_obj_size(value) == 1) { + key = flb_json_node_first(value); + if (flb_json_node_get_len(key) != 6 || + strncasecmp(flb_json_node_get_str(key), "values", 6) != 0) { + return -3; + } + value = flb_json_node_key_value(key); + } + if (type == JSON_WRAPPED_ARRAY && !flb_json_node_is_arr(value) && + (!flb_json_node_is_obj(value) || flb_json_node_obj_size(value) != 0)) { + return -2; + } + else if (type == JSON_WRAPPED_KVLIST && !flb_json_node_is_arr(value) && + !flb_json_node_is_obj(value)) { + return -2; + } + if (out_value != NULL) { + *out_value = value; + } + return 0; +} + +static struct cfl_variant *json_converted_array(struct json_cfl_builder *builder, + struct flb_json_node *array, size_t depth) +{ + size_t index; + size_t maximum; + struct flb_json_node *entry; + struct cfl_array *result; + struct cfl_variant *variant; + + if (depth > OTEL_JSON_CFL_MAX_DEPTH) { + builder->unsupported = FLB_TRUE; + return NULL; + } + if (json_cfl_account(builder, OTEL_JSON_CFL_ARRAY_HEADER_SIZE) != 0) { + return NULL; + } + result = cfl_array_create_in(builder->arena, flb_json_node_arr_size(array)); + if (result == NULL) { + return NULL; + } + flb_json_node_arr_foreach(array, index, maximum, entry) { + variant = json_converted_value(builder, entry, depth + 1); + if (variant == NULL || cfl_array_append(result, variant) != 0) { + if (variant != NULL) { + cfl_variant_destroy(variant); + } + cfl_array_destroy(result); + return NULL; + } + } + variant = cfl_variant_create_from_array_in(builder->arena, result); + if (variant == NULL) { + cfl_array_destroy(result); + } + return variant; +} + +static struct cfl_variant *json_kvlist_variant(struct json_cfl_builder *builder, + struct cfl_kvlist *kvlist) +{ + struct cfl_variant *variant; + + if (kvlist == NULL) { + return NULL; + } + variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); + if (variant == NULL) { + cfl_kvlist_destroy(kvlist); + } + return variant; +} + +/* Insert a converted key, CFL only holds string keys */ +static int json_converted_insert(struct json_cfl_builder *builder, struct cfl_kvlist *kvlist, + struct flb_json_node *key, struct cfl_variant *value, size_t depth) +{ + int result; + size_t size; + struct cfl_variant *key_variant; + + if (value == NULL) { + return -1; + } + if (flb_json_node_is_str(key)) { + return json_cfl_insert(builder, kvlist, flb_json_node_get_str(key), + flb_json_node_get_len(key), value); + } + + /* the key is accounted once, by the insertion */ + size = builder->size; + key_variant = json_converted_value(builder, key, depth + 1); + builder->size = size; + if (key_variant == NULL || key_variant->type != CFL_VARIANT_STRING) { + if (key_variant != NULL) { + builder->unsupported = FLB_TRUE; + cfl_variant_destroy(key_variant); + } + cfl_variant_destroy(value); + return -1; + } + result = json_cfl_insert(builder, kvlist, key_variant->data.as_string, + cfl_variant_size_get(key_variant), value); + cfl_variant_destroy(key_variant); + return result; +} + +static struct cfl_kvlist *json_converted_kvlist(struct json_cfl_builder *builder, + struct flb_json_node *array, size_t depth) +{ + size_t index; + size_t maximum; + struct flb_json_node *entry; + struct flb_json_node *key; + struct flb_json_node *value; + struct cfl_kvlist *result; + struct cfl_variant *variant; + + if (!flb_json_node_is_arr(array)) { + return NULL; + } + if (depth > OTEL_JSON_CFL_MAX_DEPTH) { + builder->unsupported = FLB_TRUE; + return NULL; + } + result = json_cfl_kvlist_create(builder); + if (result == NULL) { + return NULL; + } + flb_json_node_arr_foreach(array, index, maximum, entry) { + key = json_obj_get(entry, "key"); + if (!flb_json_node_is_obj(entry) || key == NULL) { + cfl_kvlist_destroy(result); + return NULL; + } + value = json_obj_get(entry, "value"); + if (value == NULL) { + /* an unset value is an empty string */ + variant = json_cfl_string(builder, "", 0); + } + else if (flb_json_node_is_obj(value) && flb_json_node_obj_size(value) == 0) { + variant = json_cfl_null(builder); + } + else if (!flb_json_node_is_obj(value)) { + cfl_kvlist_destroy(result); + return NULL; + } + else { + variant = json_converted_value(builder, value, depth + 1); + } + if (json_converted_insert(builder, result, key, variant, depth) != 0) { + cfl_kvlist_destroy(result); + return NULL; + } + } + return result; +} + +static struct cfl_variant *json_converted_map(struct json_cfl_builder *builder, + struct flb_json_node *object, size_t depth); + +/* + * Returns 0 with the converted value, NULL on failure, or the json_wrapped() + * status when the object is not a valid wrapper. + */ +static int json_unwrapped_value(struct json_cfl_builder *builder, struct flb_json_node *object, + size_t depth, struct cfl_variant **out_variant) +{ + int ret; + int type; + size_t length; + size_t bytes_size; + unsigned char *bytes; + char buffer[33]; + struct flb_json_node *value; + struct cfl_array *array; + + ret = json_wrapped(object, &value, &type); + if (ret != 0) { + return ret; + } + *out_variant = NULL; + + switch (type) { + case JSON_WRAPPED_INT: + if (flb_json_node_is_str(value)) { + length = flb_json_node_get_len(value); + if (length > sizeof(buffer) - 1) { + length = sizeof(buffer) - 1; + } + memcpy(buffer, flb_json_node_get_str(value), length); + buffer[length] = '\0'; + *out_variant = json_cfl_int(builder, strtoll(buffer, NULL, 10)); + return 0; + } + break; + case JSON_WRAPPED_BYTES: + if (json_base64_decode(flb_json_node_get_str(value), flb_json_node_get_len(value), + &bytes, &bytes_size) != 0) { + return -2; + } + if (json_cfl_account_sized(builder, json_cfl_binary_header_size(bytes_size), + bytes_size) == 0) { + *out_variant = cfl_variant_create_from_bytes_in(builder->arena, (char *) bytes, + bytes_size, CFL_FALSE); + } + flb_free(bytes); + return 0; + case JSON_WRAPPED_ARRAY: + if (flb_json_node_is_arr(value)) { + *out_variant = json_converted_array(builder, value, depth + 1); + return 0; + } + if (json_cfl_account(builder, OTEL_JSON_CFL_ARRAY_HEADER_SIZE) != 0) { + return 0; + } + array = cfl_array_create_in(builder->arena, 0); + if (array != NULL) { + *out_variant = cfl_variant_create_from_array_in(builder->arena, array); + if (*out_variant == NULL) { + cfl_array_destroy(array); + } + } + return 0; + case JSON_WRAPPED_KVLIST: + if (flb_json_node_is_arr(value)) { + *out_variant = json_kvlist_variant(builder, + json_converted_kvlist(builder, value, + depth + 1)); + } + else { + *out_variant = json_converted_map(builder, value, depth + 1); + } + return 0; + default: + break; + } + + /* strings, booleans, numbers and doubles keep their JSON value */ + *out_variant = json_converted_value(builder, value, depth + 1); + return 0; +} + +static struct cfl_variant *json_converted_map(struct json_cfl_builder *builder, + struct flb_json_node *object, size_t depth) +{ + int ret; + size_t index; + size_t maximum; + struct flb_json_node *key; + struct flb_json_node *value; + struct cfl_kvlist *kvlist; + struct cfl_variant *variant; + + if (depth > OTEL_JSON_CFL_MAX_DEPTH) { + builder->unsupported = FLB_TRUE; + return NULL; + } + variant = NULL; + ret = json_unwrapped_value(builder, object, depth, &variant); + if (ret == 0) { + return variant; + } + else if (ret != -1) { + return NULL; + } + + kvlist = json_cfl_kvlist_create(builder); + if (kvlist == NULL) { + return NULL; + } + flb_json_node_obj_foreach(object, index, maximum, key, value) { + if (json_converted_insert(builder, kvlist, key, + json_converted_value(builder, value, depth + 1), + depth) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + } + return json_kvlist_variant(builder, kvlist); +} + +static struct cfl_variant *json_converted_value(struct json_cfl_builder *builder, + struct flb_json_node *value, size_t depth) +{ + switch (flb_json_node_type(value)) { + case FLB_JSON_NODE_OBJ: + return json_converted_map(builder, value, depth + 1); + case FLB_JSON_NODE_ARR: + return json_converted_array(builder, value, depth + 1); + case FLB_JSON_NODE_NULL: + /* the established path encodes JSON null as an empty string */ + return json_cfl_string(builder, "", 0); + default: + return json_plain_value(builder, value); + } +} + +/* Copy a JSON value as is, the way the established path copies msgpack objects */ +static struct cfl_variant *json_raw_value(struct json_cfl_builder *builder, + struct flb_json_node *value, size_t depth) +{ + int type; + size_t index; + size_t maximum; + struct flb_json_node *key; + struct flb_json_node *entry; + struct cfl_array *array; + struct cfl_kvlist *kvlist; + struct cfl_variant *variant; + + if (depth > OTEL_JSON_CFL_MAX_DEPTH) { + return NULL; + } + type = flb_json_node_type(value); + if (type == FLB_JSON_NODE_OBJ) { + if (json_cfl_account(builder, + json_cfl_array_header_size(flb_json_node_obj_size(value))) != 0) { + return NULL; + } + kvlist = cfl_kvlist_create_in(builder->arena); + if (kvlist == NULL) { + return NULL; + } + flb_json_node_obj_foreach(value, index, maximum, key, entry) { + if (json_cfl_insert(builder, kvlist, flb_json_node_get_str(key), + flb_json_node_get_len(key), + json_raw_value(builder, entry, depth + 1)) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + } + variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); + if (variant == NULL) { + cfl_kvlist_destroy(kvlist); + } + return variant; + } + if (type == FLB_JSON_NODE_ARR) { + if (json_cfl_account(builder, + json_cfl_array_header_size(flb_json_node_arr_size(value))) != 0) { + return NULL; + } + array = cfl_array_create_in(builder->arena, flb_json_node_arr_size(value)); + if (array == NULL) { + return NULL; + } + flb_json_node_arr_foreach(value, index, maximum, entry) { + variant = json_raw_value(builder, entry, depth + 1); + if (variant == NULL || cfl_array_append(array, variant) != 0) { + if (variant != NULL) { + cfl_variant_destroy(variant); + } + cfl_array_destroy(array); + return NULL; + } + } + variant = cfl_variant_create_from_array_in(builder->arena, array); + if (variant == NULL) { + cfl_array_destroy(array); + } + return variant; + } + return json_plain_value(builder, value); +} + +/* + * Parse a decimal string the way the established path does: at most 31 + * characters are considered, and they must all be digits. Values beyond + * 64 bits saturate. Returns the number of digits, or -1. + */ +static int json_parse_digits(struct flb_json_node *value, uint64_t *result) +{ + size_t index; + size_t length; + const char *string; + char buffer[32]; + + if (!flb_json_node_is_str(value)) { + return -1; + } + string = flb_json_node_get_str(value); + length = flb_json_node_get_len(value); + if (length > sizeof(buffer) - 1) { + length = sizeof(buffer) - 1; + } + for (index = 0; index < length && string[index] != '\0'; index++) { + if (!isdigit((unsigned char) string[index])) { + return -1; + } + buffer[index] = string[index]; + } + buffer[index] = '\0'; + *result = index > 0 ? strtoull(buffer, NULL, 10) : 0; + return index; +} + +/* The timestamp of a log record, it must be an unsigned integer or digits */ +static int json_record_timestamp(struct flb_json_node *value, struct flb_time *tm) +{ + uint64_t number; + + if (flb_json_node_is_uint(value)) { + number = flb_json_node_get_uint(value); + } + else if (json_parse_digits(value, &number) < 0) { + return -1; + } + return flb_time_from_uint64(tm, number); +} + +/* A non-empty identifier must have the exact hexadecimal length */ +static int json_identifier_valid(struct flb_json_node *value, size_t length) +{ + size_t index; + const char *string; + + if (!flb_json_node_is_str(value)) { + return FLB_FALSE; + } + if (flb_json_node_get_len(value) == 0) { + return FLB_TRUE; + } + if (flb_json_node_get_len(value) != length) { + return FLB_FALSE; + } + string = flb_json_node_get_str(value); + for (index = 0; index < length; index++) { + if (!isxdigit((unsigned char) string[index])) { + return FLB_FALSE; + } + } + return FLB_TRUE; +} + +/* + * Reject a log record the established path rejects, with the same status and + * in the same order, before any of it is built. + */ +static int json_record_validate(struct json_cfl_builder *builder, struct flb_json_node *log_record, + struct flb_time *tm) +{ + struct flb_json_node *value; + + if (!flb_json_node_is_obj(log_record)) { + return FLB_OTEL_LOGS_ERR_UNEXPECTED_LOGRECORDS_ENTRY_TYPE; + } + value = json_obj_get(log_record, "timeUnixNano"); + if (value == NULL) { + value = json_obj_get(log_record, "observedTimeUnixNano"); + } + if (value == NULL) { + flb_time_get(tm); + } + else if (json_record_timestamp(value, tm) != 0) { + return FLB_OTEL_LOGS_ERR_UNEXPECTED_TIMESTAMP_TYPE; + } + else if (flb_time_is_valid_eventtime(tm) != FLB_TRUE) { + /* the msgpack path keeps such a record with an unrelated timestamp */ + builder->unsupported = FLB_TRUE; + return FLB_OTEL_LOGS_ERR_UNEXPECTED_TIMESTAMP_TYPE; + } + value = json_obj_get(log_record, "attributes"); + if (value != NULL && !flb_json_node_is_arr(value)) { + return FLB_OTEL_LOGS_ERR_UNEXPECTED_ATTRIBUTES_TYPE; + } + value = json_obj_get(log_record, "traceId"); + if (value != NULL && json_identifier_valid(value, 32) == FLB_FALSE) { + return FLB_OTEL_LOGS_ERR_INVALID_TRACE_ID; + } + value = json_obj_get(log_record, "spanId"); + if (value != NULL && json_identifier_valid(value, 16) == FLB_FALSE) { + return FLB_OTEL_LOGS_ERR_INVALID_SPAN_ID; + } + value = json_obj_get(log_record, "body"); + if (value != NULL && !flb_json_node_is_obj(value)) { + return FLB_OTEL_LOGS_ERR_UNEXPECTED_BODY_TYPE; + } + return 0; +} + +static struct cfl_variant *json_plain_value(struct json_cfl_builder *builder, + struct flb_json_node *value) +{ + size_t length; + const char *string; + + switch (flb_json_node_type(value)) { + case FLB_JSON_NODE_STR: + string = flb_json_node_get_strn(value, &length); + return json_cfl_string(builder, string, length); + case FLB_JSON_NODE_UINT: + return json_cfl_uint(builder, flb_json_node_get_uint(value)); + case FLB_JSON_NODE_SINT: + return json_cfl_int(builder, flb_json_node_get_sint(value)); + case FLB_JSON_NODE_REAL: + return json_cfl_double(builder, flb_json_node_get_real(value)); + case FLB_JSON_NODE_BOOL: + return json_cfl_bool(builder, flb_json_node_get_bool(value)); + case FLB_JSON_NODE_NULL: + return json_cfl_null(builder); + default: + return NULL; + } +} + +static struct cfl_kvlist *json_record_metadata(struct json_cfl_builder *builder, + struct flb_json_node *log_record) +{ + int result; + uint64_t number; + unsigned char identifier[16]; + struct flb_json_node *value; + struct cfl_kvlist *metadata; + struct cfl_kvlist *otlp; + + result = 0; + metadata = json_cfl_kvlist_create(builder); + otlp = json_cfl_kvlist_create(builder); + if (metadata == NULL || otlp == NULL) { + goto error; + } + value = json_obj_get(log_record, "observedTimeUnixNano"); + if (json_parse_digits(value, &number) > 0) { + /* the established path stores it as a signed integer */ + result = json_cfl_insert(builder, otlp, "observed_timestamp", 18, + json_cfl_int(builder, (int64_t) number)); + } + value = json_obj_get(log_record, "severityNumber"); + if (result == 0 && value != NULL) { + result = json_cfl_insert(builder, otlp, "severity_number", 15, + json_raw_value(builder, value, 0)); + } + value = json_obj_get(log_record, "severityText"); + if (result == 0 && flb_json_node_is_str(value)) { + result = json_cfl_insert(builder, otlp, "severity_text", 13, + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); + } + value = json_obj_get(log_record, "attributes"); + if (result == 0 && value != NULL) { + result = json_cfl_insert_kvlist(builder, otlp, "attributes", 10, + json_converted_kvlist(builder, value, 0)); + if (result != 0) { + builder->error_status = FLB_OTEL_LOGS_ERR_UNEXPECTED_ATTRIBUTES_TYPE; + } + } + value = json_obj_get(log_record, "droppedAttributesCount"); + if (result == 0 && value != NULL && + (flb_json_node_is_int(value) || flb_json_node_is_uint(value))) { + result = json_cfl_insert(builder, otlp, "dropped_attributes_count", 24, + json_plain_value(builder, value)); + } + value = json_obj_get(log_record, "traceId"); + if (result == 0 && flb_json_node_is_str(value) && flb_json_node_get_len(value) > 0) { + if (flb_json_node_get_len(value) != 32 || + flb_otel_utils_hex_to_id(flb_json_node_get_str(value), 32, identifier, 16) != 0 || + json_cfl_account_sized(builder, json_cfl_binary_header_size(16), 16) != 0) { + goto error; + } + result = json_cfl_insert(builder, otlp, "trace_id", 8, + cfl_variant_create_from_bytes_in(builder->arena, + (char *) identifier, 16, CFL_FALSE)); + } + value = json_obj_get(log_record, "spanId"); + if (result == 0 && flb_json_node_is_str(value) && flb_json_node_get_len(value) > 0) { + if (flb_json_node_get_len(value) != 16 || + flb_otel_utils_hex_to_id(flb_json_node_get_str(value), 16, identifier, 8) != 0 || + json_cfl_account_sized(builder, json_cfl_binary_header_size(8), 8) != 0) { + goto error; + } + result = json_cfl_insert(builder, otlp, "span_id", 7, + cfl_variant_create_from_bytes_in(builder->arena, + (char *) identifier, 8, CFL_FALSE)); + } + value = json_obj_get(log_record, "flags"); + if (value == NULL) { + value = json_obj_get(log_record, "traceFlags"); + } + if (result == 0 && (flb_json_node_is_uint(value) || json_parse_digits(value, &number) > 0)) { + if (flb_json_node_is_uint(value)) { + number = flb_json_node_get_uint(value); + } + result = json_cfl_insert(builder, otlp, "trace_flags", 11, + json_cfl_uint(builder, number & 0xff)); + } + value = json_obj_get(log_record, "eventName"); + if (result == 0 && flb_json_node_is_str(value)) { + result = json_cfl_insert(builder, otlp, "event_name", 10, + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); + } + if (result == 0) { + result = json_cfl_insert_kvlist(builder, metadata, "otlp", 4, otlp); + otlp = NULL; + } + if (result == 0) { + return metadata; + } + +error: + cfl_kvlist_destroy(otlp); + cfl_kvlist_destroy(metadata); + return NULL; +} + +static struct cfl_kvlist *json_record_body(struct json_cfl_builder *builder, + struct flb_json_node *log_record) +{ + int type; + const char *key; + struct flb_json_node *body; + struct cfl_kvlist *result; + struct cfl_variant *variant; + + body = json_obj_get(log_record, "body"); + if (body == NULL) { + return json_cfl_kvlist_create(builder); + } + + /* + * A kvlist, or a map that is not a wrapped value, is the record itself. + * Anything else is stored under the body key. + */ + type = -1; + if (json_wrapped(body, NULL, &type) != 0 || + (builder->options->body_key == NULL && type == JSON_WRAPPED_KVLIST)) { + variant = json_converted_value(builder, body, 0); + if (variant == NULL || variant->type != CFL_VARIANT_KVLIST) { + if (variant != NULL) { + cfl_variant_destroy(variant); + } + builder->error_status = FLB_OTEL_LOGS_ERR_APPEND_BODY_FAILURE; + return NULL; + } + result = variant->data.as_kvlist; + variant->data.as_kvlist = NULL; + variant->type = CFL_VARIANT_NULL; + cfl_variant_destroy(variant); + return result; + } + + result = json_cfl_kvlist_create(builder); + if (result == NULL) { + return NULL; + } + key = builder->options->body_key != NULL ? builder->options->body_key : "log"; + variant = json_converted_value(builder, body, 0); + if (json_cfl_insert(builder, result, key, strlen(key), variant) != 0) { + builder->error_status = FLB_OTEL_LOGS_ERR_APPEND_BODY_FAILURE; + cfl_kvlist_destroy(result); + return NULL; + } + return result; +} + +static struct cfl_kvlist *json_group_metadata(struct json_cfl_builder *builder, + size_t resource_index, size_t scope_index) +{ + struct cfl_kvlist *result; + + result = json_cfl_kvlist_create(builder); + if (result == NULL || + json_cfl_insert(builder, result, "schema", 6, + json_cfl_string(builder, "otlp", 4)) != 0 || + json_cfl_insert(builder, result, "resource_id", 11, + json_cfl_uint(builder, resource_index)) != 0 || + json_cfl_insert(builder, result, "scope_id", 8, + json_cfl_uint(builder, scope_index)) != 0) { + cfl_kvlist_destroy(result); + return NULL; + } + return result; +} + +static struct cfl_kvlist *json_group_body(struct json_cfl_builder *builder, + struct flb_json_node *resource_log, + struct flb_json_node *scope_log) +{ + int result; + struct flb_json_node *resource; + struct flb_json_node *scope; + struct flb_json_node *schema_url; + struct flb_json_node *value; + struct cfl_kvlist *body; + struct cfl_kvlist *resource_map; + struct cfl_kvlist *scope_map; + + result = 0; + scope_map = NULL; + body = json_cfl_kvlist_create(builder); + resource_map = json_cfl_kvlist_create(builder); + if (body == NULL || resource_map == NULL) { + goto error; + } + resource = json_obj_get(resource_log, "resource"); + if (flb_json_node_is_obj(resource)) { + value = json_obj_get(resource, "attributes"); + if (value != NULL) { + result = json_cfl_insert_kvlist(builder, resource_map, "attributes", 10, + json_converted_kvlist(builder, value, 0)); + if (result != 0) { + builder->error_status = FLB_OTEL_RESOURCE_INVALID_ATTRIBUTE; + } + } + value = json_obj_get(resource, "droppedAttributesCount"); + if (result == 0 && value != NULL) { + result = json_cfl_insert(builder, resource_map, "dropped_attributes_count", 24, + json_raw_value(builder, value, 0)); + } + } + value = json_obj_get(resource_log, "schemaUrl"); + if (result == 0 && flb_json_node_is_str(value)) { + result = json_cfl_insert(builder, resource_map, "schema_url", 10, + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); + } + if (result == 0) { + result = json_cfl_insert_kvlist(builder, body, "resource", 8, resource_map); + resource_map = NULL; + } + scope = json_obj_get(scope_log, "scope"); + if (!flb_json_node_is_obj(scope)) { + scope = NULL; + } + schema_url = json_obj_get(scope_log, "schemaUrl"); + if (!flb_json_node_is_str(schema_url)) { + schema_url = NULL; + } + + /* the scope is only described when the request carries one */ + if (scope == NULL && schema_url == NULL) { + if (result == 0) { + return body; + } + goto error; + } + if (result == 0) { + scope_map = json_cfl_kvlist_create(builder); + if (scope_map == NULL) { + goto error; + } + } + if (result == 0 && scope != NULL) { + value = json_obj_get(scope, "name"); + if (flb_json_node_is_str(value)) { + result = json_cfl_insert(builder, scope_map, "name", 4, + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); + } + value = json_obj_get(scope, "version"); + if (result == 0 && flb_json_node_is_str(value)) { + result = json_cfl_insert(builder, scope_map, "version", 7, + json_cfl_string(builder, flb_json_node_get_str(value), + flb_json_node_get_len(value))); + } + value = json_obj_get(scope, "droppedAttributesCount"); + if (result == 0 && value != NULL) { + result = json_cfl_insert(builder, scope_map, "dropped_attributes_count", 24, + json_raw_value(builder, value, 0)); + } + value = json_obj_get(scope, "attributes"); + if (result == 0 && value != NULL) { + result = json_cfl_insert_kvlist(builder, scope_map, "attributes", 10, + json_converted_kvlist(builder, value, 0)); + if (result != 0) { + builder->error_status = FLB_OTEL_LOGS_ERR_SCOPE_KVLIST; + } + } + } + if (result == 0 && schema_url != NULL) { + result = json_cfl_insert(builder, scope_map, "schema_url", 10, + json_cfl_string(builder, flb_json_node_get_str(schema_url), + flb_json_node_get_len(schema_url))); + } + if (result == 0) { + result = json_cfl_insert_kvlist(builder, body, "scope", 5, scope_map); + scope_map = NULL; + } + if (result == 0) { + return body; + } + +error: + cfl_kvlist_destroy(scope_map); + cfl_kvlist_destroy(resource_map); + cfl_kvlist_destroy(body); + return NULL; +} + +static int json_scope_logs(struct json_cfl_builder *builder, + struct flb_json_node *resource_log, struct flb_json_node *scope_log, + size_t resource_index, size_t scope_index, + size_t *record_count) +{ + size_t index; + size_t maximum; + struct flb_json_node *records; + struct flb_json_node *log_record; + struct flb_time tm; + struct flb_mp_chunk_record *group; + struct flb_mp_chunk_record *record; + + records = json_obj_get(scope_log, "logRecords"); + if (records == NULL || + (flb_json_node_is_arr(records) && flb_json_node_arr_size(records) == 0)) { + return 0; + } + flb_time_set(&tm, FLB_LOG_EVENT_GROUP_START, 0); + group = json_cfl_record_add(builder, &tm); + if (group == NULL) { + return -1; + } + group->cobj_metadata = json_cfl_object(builder, + json_group_metadata(builder, resource_index, scope_index)); + group->cobj_record = json_cfl_object(builder, + json_group_body(builder, resource_log, scope_log)); + if (group->cobj_metadata == NULL || group->cobj_record == NULL) { + return -1; + } + if (!flb_json_node_is_arr(records)) { + builder->error_status = FLB_OTEL_LOGS_ERR_UNEXPECTED_LOGRECORDS_TYPE; + return -1; + } + group->cobj_group_metadata = group->cobj_metadata; + group->cobj_group_attributes = group->cobj_record; + group->owns_group_metadata = FLB_TRUE; + group->owns_group_attributes = FLB_TRUE; + + flb_json_node_arr_foreach(records, index, maximum, log_record) { + builder->error_status = json_record_validate(builder, log_record, &tm); + if (builder->error_status != 0) { + return -1; + } + record = json_cfl_record_add(builder, &tm); + if (record == NULL) { + return -1; + } + /* the metadata is checked first, it decides the status of a bad record */ + record->cobj_metadata = json_cfl_object(builder, + json_record_metadata(builder, log_record)); + if (record->cobj_metadata == NULL) { + return -1; + } + record->cobj_record = json_cfl_object(builder, json_record_body(builder, log_record)); + if (record->cobj_record == NULL) { + return -1; + } + record->cobj_group_metadata = group->cobj_group_metadata; + record->cobj_group_attributes = group->cobj_group_attributes; + (*record_count)++; + } + + flb_time_set(&tm, FLB_LOG_EVENT_GROUP_END, 0); + record = json_cfl_record_add(builder, &tm); + if (record == NULL) { + return -1; + } + record->cobj_metadata = json_cfl_object(builder, json_cfl_kvlist_create(builder)); + record->cobj_record = json_cfl_object(builder, json_cfl_kvlist_create(builder)); + if (record->cobj_metadata == NULL || record->cobj_record == NULL) { + return -1; + } + record->cobj_group_metadata = group->cobj_group_metadata; + record->cobj_group_attributes = group->cobj_group_attributes; + return 0; +} + +int flb_opentelemetry_logs_json_to_cfl( + const char *body, size_t length, + const struct flb_opentelemetry_logs_decode_options *options, + struct flb_opentelemetry_logs_json_cfl **out_result, + int *error_status) +{ + int result; + int status; + size_t resource_index; + size_t resource_maximum; + size_t scope_index; + size_t scope_maximum; + struct flb_json_node *root; + struct flb_json_node *resource_logs; + struct flb_json_node *resource_log; + struct flb_json_node *scope_logs; + struct flb_json_node *scope_log; + struct flb_json_node *resource; + struct json_cfl_builder builder; + struct flb_opentelemetry_logs_json_cfl *output; + + status = 0; + builder.error_status = 0; + builder.unsupported = FLB_FALSE; + + if (error_status != NULL) { + *error_status = 0; + } + if (body == NULL || options == NULL || options->metadata_key == NULL || + out_result == NULL) { + return -1; + } + *out_result = NULL; + output = flb_calloc(1, sizeof(*output)); + if (output == NULL) { + return -1; + } + output->arena = json_arena_create(); + if (output->arena == NULL) { + result = -1; + goto error; + } + + /* + * The tree replaces invalid Unicode like the established conversion does. + * Leave every payload it does not read, such as content after the request + * object or a build without a tree backend, to the established conversion + * so those requests are handled exactly as before. + */ + output->document = flb_json_tree_read(body, length); + if (output->document == NULL) { + result = FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED; + goto error; + } + root = flb_json_tree_root(output->document); + if (!flb_json_node_is_obj(root)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_ROOT_OBJECT_TYPE; + result = -1; + goto error; + } + resource_logs = json_obj_get(root, "resourceLogs"); + if (resource_logs == NULL) { + status = FLB_OTEL_LOGS_ERR_RESOURCELOGS_MISSING; + result = -1; + goto error; + } + if (!flb_json_node_is_arr(resource_logs)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_RESOURCELOGS_TYPE; + result = -1; + goto error; + } + builder.options = options; + builder.arena = output->arena; + builder.size = 0; + builder.limit = options->output_limit; + builder.chunk = flb_mp_chunk_cobj_create(NULL, NULL); + if (builder.chunk == NULL) { + result = -1; + goto error; + } + builder.chunk->record_pool_enabled = FLB_TRUE; + output->chunk = builder.chunk; + + /* the checks and their order follow flb_opentelemetry_logs_json_to_msgpack() */ + flb_json_node_arr_foreach(resource_logs, resource_index, resource_maximum, resource_log) { + if (!flb_json_node_is_obj(resource_log)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_RESOURCELOGS_ENTRY_TYPE; + result = -1; + goto error; + } + resource = json_obj_get(resource_log, "resource"); + if (flb_json_node_is_obj(resource) && json_obj_get(resource, "attributes") != NULL && + !flb_json_node_is_arr(json_obj_get(resource, "attributes"))) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_ATTRIBUTES_TYPE; + result = -1; + goto error; + } + scope_logs = json_obj_get(resource_log, "scopeLogs"); + if (scope_logs == NULL) { + status = FLB_OTEL_LOGS_ERR_SCOPELOGS_MISSING; + result = -1; + goto error; + } + if (!flb_json_node_is_arr(scope_logs)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_SCOPELOGS_TYPE; + result = -1; + goto error; + } + flb_json_node_arr_foreach(scope_logs, scope_index, scope_maximum, scope_log) { + if (!flb_json_node_is_obj(scope_log)) { + status = FLB_OTEL_LOGS_ERR_UNEXPECTED_SCOPELOGS_TYPE; + result = -1; + goto error; + } + if (json_scope_logs(&builder, resource_log, scope_log, + resource_index, scope_index, + &output->record_count) != 0) { + status = builder.error_status; + result = -1; + if (builder.unsupported == FLB_TRUE) { + status = 0; + result = FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED; + } + goto error; + } + } + } + output->encoded_size = builder.size; + *out_result = output; + return 0; + +error: + if (error_status != NULL) { + *error_status = status; + } + flb_opentelemetry_logs_json_cfl_destroy(output); + return result; +} + +struct flb_mp_chunk_cobj *flb_opentelemetry_logs_json_cfl_take_chunk( + struct flb_opentelemetry_logs_json_cfl *result) +{ + struct flb_mp_chunk_cobj *chunk; + + if (result == NULL) { + return NULL; + } + chunk = result->chunk; + result->chunk = NULL; + return chunk; +} + +size_t flb_opentelemetry_logs_json_cfl_get_encoded_size( + struct flb_opentelemetry_logs_json_cfl *result) +{ + return result != NULL ? result->encoded_size : 0; +} + +void flb_opentelemetry_logs_json_cfl_destroy( + struct flb_opentelemetry_logs_json_cfl *result) +{ + if (result == NULL) { + return; + } + flb_mp_chunk_cobj_destroy(result->chunk); + cfl_arena_destroy(result->arena); + flb_json_tree_destroy(result->document); + flb_free(result); +} diff --git a/src/opentelemetry/flb_opentelemetry_logs_proto.c b/src/opentelemetry/flb_opentelemetry_logs_proto.c new file mode 100644 index 00000000000..e91050fa824 --- /dev/null +++ b/src/opentelemetry/flb_opentelemetry_logs_proto.c @@ -0,0 +1,1740 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +/* + * Small requests use protobuf-c's default allocator to avoid reserving an + * arena chunk. Realistic batched requests use the arena to amortize the large + * number of short-lived allocations performed by protobuf-c. + */ +#define OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE 8192 +#define OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE 4096 +#define OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE 65536 + +static void *protobuf_arena_chunk_malloc(void *context, size_t size) +{ + (void) context; + + return flb_malloc(size); +} +static void protobuf_arena_chunk_free(void *context, void *pointer) +{ + (void) context; + + flb_free(pointer); +} + +static void *protobuf_arena_alloc(void *allocator_data, size_t size) +{ + struct cfl_arena *arena; + + arena = allocator_data; + if (size == 0) { + size = 1; + } + + return cfl_arena_malloc(arena, size); +} + +static void protobuf_arena_free(void *allocator_data, void *pointer) +{ + (void) allocator_data; + (void) pointer; +} + +static struct cfl_arena *protobuf_arena_create(void) +{ + struct cfl_arena_options options; + + cfl_arena_options_init(&options); + options.chunk_size = OTEL_PROTOBUF_ARENA_INITIAL_CHUNK_SIZE; + options.maximum_chunk_size = OTEL_PROTOBUF_ARENA_MAX_CHUNK_SIZE; + options.malloc_fn = protobuf_arena_chunk_malloc; + options.free_fn = protobuf_arena_chunk_free; + + return cfl_arena_create_with_options(&options); +} + +static Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest * +protobuf_logs_unpack(ProtobufCAllocator *allocator, size_t size, const uint8_t *data) +{ + if (flb_opentelemetry_protobuf_validate( + &opentelemetry__proto__collector__logs__v1__export_logs_service_request__descriptor, + data, size) != 0) { + return NULL; + } + return opentelemetry__proto__collector__logs__v1__export_logs_service_request__unpack( + allocator, size, data); +} + +static void protobuf_logs_free( + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *logs, + ProtobufCAllocator *allocator) +{ + /* + * Every allocation of an arena backed request is released with the arena, + * walking the message tree here would only call the no-op free callback. + */ + if (allocator != NULL && allocator->free == protobuf_arena_free) { + return; + } + + opentelemetry__proto__collector__logs__v1__export_logs_service_request__free_unpacked( + logs, allocator); +} + +/* Unpack a logs request, using an arena for realistic batched payloads */ +static Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest * +protobuf_logs_decode(uint8_t *in_buf, size_t in_size, + struct cfl_arena **protobuf_arena, + ProtobufCAllocator *arena_allocator, + ProtobufCAllocator **protobuf_allocator) +{ + *protobuf_arena = NULL; + *protobuf_allocator = NULL; + + if (in_size >= OTEL_PROTOBUF_ARENA_MIN_PAYLOAD_SIZE) { + *protobuf_arena = protobuf_arena_create(); + if (*protobuf_arena != NULL) { + arena_allocator->alloc = protobuf_arena_alloc; + arena_allocator->free = protobuf_arena_free; + arena_allocator->allocator_data = *protobuf_arena; + *protobuf_allocator = arena_allocator; + } + } + + return protobuf_logs_unpack(*protobuf_allocator, in_size, in_buf); +} + +/* + * OTLP encoding functions to pack the log records as msgpack + * ---------------------------------------------------------- + */ +static int otlp_pack_any_value(msgpack_packer *mp_pck, Opentelemetry__Proto__Common__V1__AnyValue *body); + +static int otel_pack_string(msgpack_packer *mp_pck, char *str) +{ + return msgpack_pack_str_with_body(mp_pck, str, strlen(str)); +} + +static int otel_pack_bool(msgpack_packer *mp_pck, bool val) +{ + if (val) { + return msgpack_pack_true(mp_pck); + } + else { + return msgpack_pack_false(mp_pck); + } +} + +static int otel_pack_int(msgpack_packer *mp_pck, int64_t val) +{ + return msgpack_pack_int64(mp_pck, val); +} + +static int otel_pack_double(msgpack_packer *mp_pck, double val) +{ + return msgpack_pack_double(mp_pck, val); +} + +static int otel_pack_kvarray(msgpack_packer *mp_pck, + Opentelemetry__Proto__Common__V1__KeyValue **kv_array, + size_t kv_count) +{ + int result; + int index; + + result = msgpack_pack_map(mp_pck, kv_count); + + if (result != 0) { + return result; + } + + for (index = 0; index < kv_count && result == 0; index++) { + result = otel_pack_string(mp_pck, kv_array[index]->key); + + if(result == 0) { + result = otlp_pack_any_value(mp_pck, kv_array[index]->value); + } + } + + return result; +} + +static int otel_pack_kvlist(msgpack_packer *mp_pck, + Opentelemetry__Proto__Common__V1__KeyValueList *kv_list) +{ + int kv_index; + int ret; + char *key; + Opentelemetry__Proto__Common__V1__AnyValue *value; + + ret = msgpack_pack_map(mp_pck, kv_list->n_values); + if (ret != 0) { + return ret; + } + + for (kv_index = 0; kv_index < kv_list->n_values && ret == 0; kv_index++) { + key = kv_list->values[kv_index]->key; + value = kv_list->values[kv_index]->value; + + ret = otel_pack_string(mp_pck, key); + + if(ret == 0) { + ret = otlp_pack_any_value(mp_pck, value); + } + } + + return ret; +} + +static int otel_pack_array(msgpack_packer *mp_pck, + Opentelemetry__Proto__Common__V1__ArrayValue *array) +{ + int ret; + int array_index; + + ret = msgpack_pack_array(mp_pck, array->n_values); + + if (ret != 0) { + return ret; + } + + for (array_index = 0; array_index < array->n_values && ret == 0; array_index++) { + ret = otlp_pack_any_value(mp_pck, array->values[array_index]); + } + + return ret; +} + +static int otel_pack_bytes(msgpack_packer *mp_pck, + ProtobufCBinaryData bytes) +{ + return msgpack_pack_bin_with_body(mp_pck, bytes.data, bytes.len); +} + +static int otlp_pack_any_value(msgpack_packer *mp_pck, + Opentelemetry__Proto__Common__V1__AnyValue *body) +{ + int result; + + result = -2; + + if (body == NULL) { + msgpack_pack_nil(mp_pck); + return 0; + } + + switch(body->value_case){ + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: + result = otel_pack_string(mp_pck, body->string_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE_STRINDEX: + /* Profiling-only string dictionary reference: ignore in logs. */ + result = msgpack_pack_nil(mp_pck); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: + result = otel_pack_bool(mp_pck, body->bool_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: + result = otel_pack_int(mp_pck, body->int_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: + result = otel_pack_double(mp_pck, body->double_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: + result = otel_pack_array(mp_pck, body->array_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: + result = otel_pack_kvlist(mp_pck, body->kvlist_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: + result = otel_pack_bytes(mp_pck, body->bytes_value); + break; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET: + /* treat an unset value as null */ + result = msgpack_pack_nil(mp_pck); + break; + + default: + break; + } + + if (result == -2) { + flb_error("[otel]: invalid value type in pack_any_value"); + result = -1; + } + + return result; +} + +/* https://opentelemetry.io/docs/specs/otel/logs/data-model/#log-and-event-record-definition */ +static int otel_pack_v1_metadata( + const struct flb_opentelemetry_logs_decode_options *options, + msgpack_packer *mp_pck, + struct Opentelemetry__Proto__Logs__V1__LogRecord *log_record, + Opentelemetry__Proto__Resource__V1__Resource *resource, + Opentelemetry__Proto__Common__V1__InstrumentationScope *scope) +{ + int ret; + int len; + struct flb_mp_map_header mh; + struct flb_mp_map_header otlp_mh; + + flb_mp_map_header_init(&otlp_mh, mp_pck); + + len = strlen(options->metadata_key); + + /* otlp key start */ + flb_mp_map_header_append(&otlp_mh); + + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, options->metadata_key, len); + + flb_mp_map_header_init(&mh, mp_pck); + + if (log_record->observed_time_unix_nano != 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 18); + msgpack_pack_str_body(mp_pck, "observed_timestamp", 18); + msgpack_pack_uint64(mp_pck, log_record->observed_time_unix_nano); + } + + /* Value of 0 indicates unknown or missing timestamp. */ + if (log_record->time_unix_nano != 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 9); + msgpack_pack_str_body(mp_pck, "timestamp", 9); + msgpack_pack_uint64(mp_pck, log_record->time_unix_nano); + } + + /* https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber */ + if (log_record->severity_number >= 1 && log_record->severity_number <= 24) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 15); + msgpack_pack_str_body(mp_pck, "severity_number", 15); + msgpack_pack_uint64(mp_pck, log_record->severity_number); + } + + if (log_record->severity_text != NULL && strlen(log_record->severity_text) > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 13); + msgpack_pack_str_body(mp_pck, "severity_text", 13); + msgpack_pack_str(mp_pck, strlen(log_record->severity_text)); + msgpack_pack_str_body(mp_pck, log_record->severity_text, strlen(log_record->severity_text)); + } + + if (log_record->n_attributes > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "attributes", 10); + ret = otel_pack_kvarray(mp_pck, + log_record->attributes, + log_record->n_attributes); + if (ret != 0) { + return ret; + } + } + + if (log_record->dropped_attributes_count > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 24); + msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); + msgpack_pack_uint64(mp_pck, log_record->dropped_attributes_count); + } + + if (log_record->trace_id.len > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 8); + msgpack_pack_str_body(mp_pck, "trace_id", 8); + ret = otel_pack_bytes(mp_pck, log_record->trace_id); + if (ret != 0) { + return ret; + } + } + + if (log_record->span_id.len > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 7); + msgpack_pack_str_body(mp_pck, "span_id", 7); + ret = otel_pack_bytes(mp_pck, log_record->span_id); + if (ret != 0) { + return ret; + } + } + + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 11); + msgpack_pack_str_body(mp_pck, "trace_flags", 11); + msgpack_pack_uint8(mp_pck, (uint8_t) log_record->flags & 0xff); + + if (log_record->event_name != NULL && strlen(log_record->event_name) > 0) { + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "event_name", 10); + msgpack_pack_str(mp_pck, strlen(log_record->event_name)); + msgpack_pack_str_body(mp_pck, log_record->event_name, strlen(log_record->event_name)); + } + + flb_mp_map_header_end(&mh); + + /* otlp key end */ + flb_mp_map_header_end(&otlp_mh); + + return 0; +} + +int flb_opentelemetry_logs_protobuf_to_msgpack( + struct flb_log_event_encoder *encoder, + const void *payload, size_t payload_size, + const struct flb_opentelemetry_logs_decode_options *options, + size_t *record_count) +{ + int ret = 0; + int len; + int resource_logs_index; + int scope_log_index; + int log_record_index; + char *logs_body_key; + int scope_has_schema_url; + struct cfl_arena *protobuf_arena; + struct flb_mp_map_header mh; + struct flb_mp_map_header mh_tmp; + struct flb_time tm; + ProtobufCAllocator arena_allocator; + ProtobufCAllocator *protobuf_allocator; + + msgpack_packer *mp_pck; + msgpack_packer *mp_pck_meta; + + /* OTel proto suff */ + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *input_logs; + Opentelemetry__Proto__Logs__V1__ScopeLogs **scope_logs; + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; + Opentelemetry__Proto__Common__V1__InstrumentationScope *scope; + + Opentelemetry__Proto__Logs__V1__ResourceLogs **resource_logs; + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; + Opentelemetry__Proto__Logs__V1__LogRecord **log_records; + Opentelemetry__Proto__Resource__V1__Resource *resource; + + mp_pck = &encoder->body.packer; + mp_pck_meta = &encoder->metadata.packer; + *record_count = 0; + + /* unpack logs from protobuf payload */ + if (encoder == NULL || options == NULL || options->metadata_key == NULL || + record_count == NULL || (payload == NULL && payload_size != 0)) { + return -1; + } + + input_logs = protobuf_logs_decode((uint8_t *) payload, payload_size, &protobuf_arena, + &arena_allocator, &protobuf_allocator); + if (input_logs == NULL) { + flb_warn("[opentelemetry] failed to unpack protobuf logs payload"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + resource_logs = input_logs->resource_logs; + if (input_logs->n_resource_logs == 0) { + ret = 0; + goto binary_payload_to_msgpack_end; + } + + if (resource_logs == NULL) { + flb_warn("[opentelemetry] no resource logs found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + for (resource_logs_index = 0; resource_logs_index < input_logs->n_resource_logs; resource_logs_index++) { + resource_log = resource_logs[resource_logs_index]; + if (resource_log == NULL) { + flb_warn("[opentelemetry] null resource logs entry found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + resource = resource_log->resource; + scope_logs = resource_log->scope_logs; + + if (resource_log->n_scope_logs > 0 && scope_logs == NULL) { + flb_warn("[opentelemetry] no scope logs found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + for (scope_log_index = 0; scope_log_index < resource_log->n_scope_logs; scope_log_index++) { + scope_log = scope_logs[scope_log_index]; + if (scope_log == NULL) { + flb_warn("[opentelemetry] null scope logs entry found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + log_records = scope_log->log_records; + + if (scope_log->n_log_records == 0) { + continue; + } + + if (log_records == NULL) { + flb_warn("[opentelemetry] no log records found"); + ret = -1; + goto binary_payload_to_msgpack_end; + } + + flb_log_event_encoder_group_init(encoder); + + /* pack schema (internal) */ + ret = flb_log_event_encoder_append_metadata_values(encoder, + FLB_LOG_EVENT_STRING_VALUE("schema", 6), + FLB_LOG_EVENT_STRING_VALUE("otlp", 4), + FLB_LOG_EVENT_STRING_VALUE("resource_id", 11), + FLB_LOG_EVENT_INT64_VALUE(resource_logs_index), + FLB_LOG_EVENT_STRING_VALUE("scope_id", 8), + FLB_LOG_EVENT_INT64_VALUE(scope_log_index)); + + + ret = flb_log_event_encoder_dynamic_field_reset(&encoder->body); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[opentelemetry] failed to reset log event body: %s", + flb_log_event_encoder_get_error_description(ret)); + goto binary_payload_to_msgpack_end; + } + + flb_mp_map_header_init(&mh, mp_pck); + + /* Resource */ + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 8); + msgpack_pack_str_body(mp_pck, "resource", 8); + + flb_mp_map_header_init(&mh_tmp, mp_pck); + if (resource) { + /* look for OTel resource attributes */ + if (resource->n_attributes > 0 && resource->attributes) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "attributes", 10); + + ret = otel_pack_kvarray(mp_pck, + resource->attributes, + resource->n_attributes); + if (ret != 0) { + goto binary_payload_to_msgpack_end; + } + } + + if (resource->dropped_attributes_count > 0) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 24); + msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); + msgpack_pack_uint64(mp_pck, resource->dropped_attributes_count); + } + + if (resource_log->schema_url) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "schema_url", 10); + + len = strlen(resource_log->schema_url); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, resource_log->schema_url, len); + } + } + flb_mp_map_header_end(&mh_tmp); + + /* scope */ + flb_mp_map_header_append(&mh); + msgpack_pack_str(mp_pck, 5); + msgpack_pack_str_body(mp_pck, "scope", 5); + + /* Scope */ + scope = scope_log->scope; + scope_has_schema_url = FLB_FALSE; + + if (scope_log->schema_url && strlen(scope_log->schema_url) > 0) { + scope_has_schema_url = FLB_TRUE; + } + + if (scope && (scope->name || scope->version || + scope->n_attributes > 0 || scope->dropped_attributes_count > 0 || + scope_has_schema_url == FLB_TRUE)) { + flb_mp_map_header_init(&mh_tmp, mp_pck); + + if (scope_has_schema_url == FLB_TRUE) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "schema_url", 10); + + len = strlen(scope_log->schema_url); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, scope_log->schema_url, len); + } + + if (scope->name && strlen(scope->name) > 0) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 4); + msgpack_pack_str_body(mp_pck, "name", 4); + + len = strlen(scope->name); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, scope->name, len); + } + if (scope->version && strlen(scope->version) > 0) { + flb_mp_map_header_append(&mh_tmp); + + msgpack_pack_str(mp_pck, 7); + msgpack_pack_str_body(mp_pck, "version", 7); + + len = strlen(scope->version); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, scope->version, len); + } + + if (scope->n_attributes > 0 && scope->attributes) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "attributes", 10); + ret = otel_pack_kvarray(mp_pck, + scope->attributes, + scope->n_attributes); + if (ret != 0) { + goto binary_payload_to_msgpack_end; + } + } + + if (scope->dropped_attributes_count > 0) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 24); + msgpack_pack_str_body(mp_pck, "dropped_attributes_count", 24); + msgpack_pack_uint64(mp_pck, scope->dropped_attributes_count); + } + + flb_mp_map_header_end(&mh_tmp); + } + else { + flb_mp_map_header_init(&mh_tmp, mp_pck); + + if (scope_has_schema_url == FLB_TRUE) { + flb_mp_map_header_append(&mh_tmp); + msgpack_pack_str(mp_pck, 10); + msgpack_pack_str_body(mp_pck, "schema_url", 10); + + len = strlen(scope_log->schema_url); + msgpack_pack_str(mp_pck, len); + msgpack_pack_str_body(mp_pck, scope_log->schema_url, len); + } + + flb_mp_map_header_end(&mh_tmp); + } + + flb_mp_map_header_end(&mh); + + ret = flb_log_event_encoder_dynamic_field_flush(&encoder->body); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[opentelemetry] could not set group content metadata: %s", + flb_log_event_encoder_get_error_description(ret)); + goto binary_payload_to_msgpack_end; + } + + ret = flb_log_event_encoder_group_header_end(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto binary_payload_to_msgpack_end; + } + + for (log_record_index=0; log_record_index < scope_log->n_log_records; log_record_index++) { + ret = flb_log_event_encoder_begin_record(encoder); + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + if (log_records[log_record_index]->time_unix_nano > 0) { + ret = flb_time_from_uint64( + &tm, + log_records[log_record_index]->time_unix_nano); + if (ret == 0) { + ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + } + } + else if (log_records[log_record_index]->observed_time_unix_nano > 0) { + ret = flb_time_from_uint64( + &tm, + log_records[log_record_index]->observed_time_unix_nano); + if (ret == 0) { + ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + } + } + else { + flb_time_get(&tm); + ret = flb_log_event_encoder_set_timestamp(encoder, &tm); + } + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_dynamic_field_reset(&encoder->metadata); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[opentelemetry] failed to reset log event metadata: %s", + flb_log_event_encoder_get_error_description(ret)); + ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; + } + else { + ret = otel_pack_v1_metadata(options, + mp_pck_meta, + log_records[log_record_index], + resource, + scope_log->scope); + } + + if (ret != 0) { + flb_error("[opentelemetry] failed to convert log record"); + ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; + } + else { + ret = flb_log_event_encoder_dynamic_field_flush(&encoder->metadata); + } + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_dynamic_field_reset(&encoder->body); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + flb_error("[opentelemetry] failed to reset log event body: %s", + flb_log_event_encoder_get_error_description(ret)); + ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; + } + else if (options->body_key == NULL && + log_records[log_record_index]->body != NULL && + log_records[log_record_index]->body->value_case == + OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { + ret = otlp_pack_any_value( + mp_pck, + log_records[log_record_index]->body); + } + else { + logs_body_key = (char *) options->body_key; + if (logs_body_key == NULL) { + logs_body_key = "log"; + } + ret = msgpack_pack_map(mp_pck, 1); + if (ret == 0) { + ret = msgpack_pack_str(mp_pck, strlen(logs_body_key)); + } + if (ret == 0) { + ret = msgpack_pack_str_body(mp_pck, + logs_body_key, + strlen(logs_body_key)); + } + if (ret == 0) { + ret = otlp_pack_any_value( + mp_pck, + log_records[log_record_index]->body); + } + } + + if (ret != 0) { + flb_error("[opentelemetry] failed to convert log record body"); + ret = FLB_EVENT_ENCODER_ERROR_SERIALIZATION_FAILURE; + } + else { + ret = flb_log_event_encoder_dynamic_field_flush(&encoder->body); + } + } + + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + ret = flb_log_event_encoder_commit_record(encoder); + if (ret == FLB_EVENT_ENCODER_SUCCESS) { + (*record_count)++; + } + else { + goto binary_payload_to_msgpack_end; + } + } + else { + flb_error("[opentelemetry] marshalling error"); + goto binary_payload_to_msgpack_end; + } + } + + ret = flb_log_event_encoder_group_end(encoder); + if (ret != FLB_EVENT_ENCODER_SUCCESS) { + goto binary_payload_to_msgpack_end; + } + + } + } + + binary_payload_to_msgpack_end: + if (input_logs) { + protobuf_logs_free(input_logs, protobuf_allocator); + } + cfl_arena_destroy(protobuf_arena); + + if (ret != 0) { + return -1; + } + + return 0; +} + +/* + * OTLP encoding functions to build the log records as CFL objects + * --------------------------------------------------------------- + * + * When the input processors can take a chunk of CFL records, the protobuf + * request is converted straight into the records the processors work on, + * skipping the msgpack buffer they would otherwise decode. The records match + * what the processors get from the msgpack encoding above: non-negative + * integers are unsigned values and strings or bytes are borrowed from the + * unpacked request, which outlives the processing. + * + * 'size' is the MessagePack size the msgpack encoding above produces for the + * same request. It enforces 'logs_max_size' exactly like that path, without + * building the intermediate MessagePack buffer: metadata and group maps use + * 32-bit headers, attribute maps, kvlists and the body map compact ones. + */ +#define OTEL_CFL_MAP_HEADER_SIZE 5 +#define OTEL_CFL_RECORD_SIZE 12 + +struct otel_cfl_builder { + const struct flb_opentelemetry_logs_decode_options *options; + struct flb_mp_chunk_cobj *chunk; + struct cfl_arena *arena; + size_t size; + size_t limit; +}; + +static int otel_cfl_account(struct otel_cfl_builder *builder, size_t bytes) +{ + if (bytes > builder->limit || builder->size > builder->limit - bytes) { + return -1; + } + + builder->size += bytes; + return 0; +} + +static size_t otel_cfl_uint_size(uint64_t value) +{ + if (value <= 0x7f) { + return 1; + } + if (value <= UINT8_MAX) { + return 2; + } + if (value <= UINT16_MAX) { + return 3; + } + if (value <= UINT32_MAX) { + return 5; + } + + return 9; +} + +static size_t otel_cfl_int_size(int64_t value) +{ + if (value >= -32) { + return 1; + } + if (value >= INT8_MIN) { + return 2; + } + if (value >= INT16_MIN) { + return 3; + } + if (value >= INT32_MIN) { + return 5; + } + + return 9; +} + +static size_t otel_cfl_string_header_size(size_t length) +{ + if (length <= 31) { + return 1; + } + if (length <= UINT8_MAX) { + return 2; + } + if (length <= UINT16_MAX) { + return 3; + } + + return 5; +} + +static size_t otel_cfl_binary_header_size(size_t length) +{ + if (length <= UINT8_MAX) { + return 2; + } + if (length <= UINT16_MAX) { + return 3; + } + + return 5; +} + +static size_t otel_cfl_array_header_size(size_t entries) +{ + if (entries <= 15) { + return 1; + } + if (entries <= UINT16_MAX) { + return 3; + } + + return 5; +} + +static int otel_cfl_account_sized_value(struct otel_cfl_builder *builder, + size_t header_size, size_t value_size) +{ + if (otel_cfl_account(builder, header_size) != 0) { + return -1; + } + + return otel_cfl_account(builder, value_size); +} + +static struct cfl_variant *otel_cfl_string(struct otel_cfl_builder *builder, + char *str, size_t len) +{ + if (otel_cfl_account_sized_value(builder, + otel_cfl_string_header_size(len), len) != 0) { + return NULL; + } + + return cfl_variant_create_from_string_s_in(builder->arena, str, len, CFL_TRUE); +} + +static struct cfl_variant *otel_cfl_bytes(struct otel_cfl_builder *builder, + ProtobufCBinaryData bytes) +{ + if (otel_cfl_account_sized_value(builder, + otel_cfl_binary_header_size(bytes.len), + bytes.len) != 0) { + return NULL; + } + + return cfl_variant_create_from_bytes_in(builder->arena, + (char *) bytes.data, bytes.len, CFL_TRUE); +} + +static struct cfl_variant *otel_cfl_uint(struct otel_cfl_builder *builder, uint64_t val) +{ + if (otel_cfl_account(builder, otel_cfl_uint_size(val)) != 0) { + return NULL; + } + + return cfl_variant_create_from_uint64_in(builder->arena, val); +} + +/* msgpack keeps non-negative integers as unsigned values */ +static struct cfl_variant *otel_cfl_int(struct otel_cfl_builder *builder, int64_t val) +{ + if (val >= 0) { + return otel_cfl_uint(builder, (uint64_t) val); + } + + if (otel_cfl_account(builder, otel_cfl_int_size(val)) != 0) { + return NULL; + } + + return cfl_variant_create_from_int64_in(builder->arena, val); +} + +/* Insert a value, the value is released when it cannot be inserted */ +static int otel_cfl_insert(struct otel_cfl_builder *builder, struct cfl_kvlist *kvlist, + const char *key, size_t key_len, struct cfl_variant *value) +{ + if (value == NULL) { + return -1; + } + + if (otel_cfl_account_sized_value(builder, + otel_cfl_string_header_size(key_len), + key_len) != 0 || + cfl_kvlist_insert_s(kvlist, (char *) key, key_len, value) != 0) { + cfl_variant_destroy(value); + return -1; + } + + return 0; +} + +/* Insert a kvlist, the kvlist is released when it cannot be inserted */ +static int otel_cfl_insert_kvlist(struct otel_cfl_builder *builder, + struct cfl_kvlist *kvlist, + const char *key, size_t key_len, + struct cfl_kvlist *value) +{ + struct cfl_variant *variant; + + if (value == NULL) { + return -1; + } + + variant = cfl_variant_create_from_kvlist_in(builder->arena, value); + if (variant == NULL) { + cfl_kvlist_destroy(value); + return -1; + } + + return otel_cfl_insert(builder, kvlist, key, key_len, variant); +} + +static struct cfl_kvlist *otel_cfl_kvlist_create(struct otel_cfl_builder *builder) +{ + /* CFL's dynamic map encoder reserves a map32 header for every kvlist. */ + if (otel_cfl_account(builder, OTEL_CFL_MAP_HEADER_SIZE) != 0) { + return NULL; + } + + return cfl_kvlist_create_in(builder->arena); +} + +/* A map the msgpack path packs with a compact header */ +static struct cfl_kvlist *otel_cfl_kvlist_create_compact(struct otel_cfl_builder *builder, + size_t entries) +{ + if (otel_cfl_account(builder, otel_cfl_array_header_size(entries)) != 0) { + return NULL; + } + + return cfl_kvlist_create_in(builder->arena); +} + +static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__AnyValue *value); + +static struct cfl_kvlist *otel_cfl_kvarray(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__KeyValue **kv_array, + size_t kv_count) +{ + size_t index; + struct cfl_kvlist *kvlist; + + kvlist = otel_cfl_kvlist_create_compact(builder, kv_count); + if (kvlist == NULL) { + return NULL; + } + + for (index = 0; index < kv_count; index++) { + if (otel_cfl_insert(builder, kvlist, + kv_array[index]->key, strlen(kv_array[index]->key), + otel_cfl_any_value(builder, kv_array[index]->value)) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + } + + return kvlist; +} + +static struct cfl_variant *otel_cfl_array(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__ArrayValue *array) +{ + size_t index; + struct cfl_array *entries; + struct cfl_variant *entry; + struct cfl_variant *variant; + + if (otel_cfl_account(builder, + otel_cfl_array_header_size(array->n_values)) != 0) { + return NULL; + } + + entries = cfl_array_create_in(builder->arena, array->n_values); + if (entries == NULL) { + return NULL; + } + + for (index = 0; index < array->n_values; index++) { + entry = otel_cfl_any_value(builder, array->values[index]); + if (entry == NULL || cfl_array_append(entries, entry) != 0) { + if (entry != NULL) { + cfl_variant_destroy(entry); + } + cfl_array_destroy(entries); + return NULL; + } + } + + variant = cfl_variant_create_from_array_in(builder->arena, entries); + if (variant == NULL) { + cfl_array_destroy(entries); + } + + return variant; +} + +static struct cfl_variant *otel_cfl_any_value(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__AnyValue *value) +{ + struct cfl_kvlist *kvlist; + struct cfl_variant *variant; + + if (value == NULL) { + if (otel_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_null_in(builder->arena); + } + + switch (value->value_case) { + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE: + return otel_cfl_string(builder, value->string_value, strlen(value->string_value)); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BOOL_VALUE: + if (otel_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_bool_in(builder->arena, value->bool_value); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_INT_VALUE: + return otel_cfl_int(builder, value->int_value); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_DOUBLE_VALUE: + if (otel_cfl_account(builder, 9) != 0) { + return NULL; + } + return cfl_variant_create_from_double_in(builder->arena, value->double_value); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_ARRAY_VALUE: + return otel_cfl_array(builder, value->array_value); + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE: + kvlist = otel_cfl_kvarray(builder, value->kvlist_value->values, + value->kvlist_value->n_values); + if (kvlist == NULL) { + return NULL; + } + + variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); + if (variant == NULL) { + cfl_kvlist_destroy(kvlist); + } + return variant; + + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_BYTES_VALUE: + return otel_cfl_bytes(builder, value->bytes_value); + + /* profiling-only string dictionary references are ignored in logs */ + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE_STRINDEX: + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET: + if (otel_cfl_account(builder, 1) != 0) { + return NULL; + } + return cfl_variant_create_from_null_in(builder->arena); + + default: + flb_error("[otel]: invalid value type in pack_any_value"); + return NULL; + } +} + +/* Wrap a kvlist as a record object, the kvlist is released on failure */ +static struct cfl_object *otel_cfl_object(struct otel_cfl_builder *builder, + struct cfl_kvlist *kvlist) +{ + struct cfl_object *object; + + if (kvlist == NULL) { + return NULL; + } + + object = cfl_object_create(); + if (object == NULL) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + + object->variant = cfl_variant_create_from_kvlist_in(builder->arena, kvlist); + if (object->variant == NULL) { + cfl_kvlist_destroy(kvlist); + cfl_object_destroy(object); + return NULL; + } + object->type = CFL_OBJECT_KVLIST; + + return object; +} + +/* Append a record to the chunk, its objects are owned by the chunk */ +static struct flb_mp_chunk_record *otel_cfl_record_add(struct otel_cfl_builder *builder, + struct flb_time *tm) +{ + struct flb_mp_chunk_record *record; + + if (otel_cfl_account(builder, OTEL_CFL_RECORD_SIZE) != 0) { + return NULL; + } + + record = flb_mp_chunk_record_create(builder->chunk); + if (record == NULL) { + return NULL; + } + + memset(&record->event, 0, sizeof(record->event)); + flb_time_copy(&record->event.timestamp, tm); + cfl_list_add(&record->_head, &builder->chunk->records); + + return record; +} + +/* https://opentelemetry.io/docs/specs/otel/logs/data-model/#log-and-event-record-definition */ +static struct cfl_kvlist *otel_cfl_v1_metadata( + struct otel_cfl_builder *builder, + Opentelemetry__Proto__Logs__V1__LogRecord *log_record) +{ + int ret = 0; + struct cfl_kvlist *otlp; + struct cfl_kvlist *metadata; + + metadata = otel_cfl_kvlist_create(builder); + otlp = otel_cfl_kvlist_create(builder); + if (metadata == NULL || otlp == NULL) { + goto error; + } + + if (log_record->observed_time_unix_nano != 0) { + ret = otel_cfl_insert(builder, otlp, "observed_timestamp", 18, + otel_cfl_uint(builder, log_record->observed_time_unix_nano)); + } + + /* Value of 0 indicates unknown or missing timestamp. */ + if (ret == 0 && log_record->time_unix_nano != 0) { + ret = otel_cfl_insert(builder, otlp, "timestamp", 9, + otel_cfl_uint(builder, log_record->time_unix_nano)); + } + + /* https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber */ + if (ret == 0 && log_record->severity_number >= 1 && log_record->severity_number <= 24) { + ret = otel_cfl_insert(builder, otlp, "severity_number", 15, + otel_cfl_uint(builder, log_record->severity_number)); + } + + if (ret == 0 && log_record->severity_text != NULL && + strlen(log_record->severity_text) > 0) { + ret = otel_cfl_insert(builder, otlp, "severity_text", 13, + otel_cfl_string(builder, log_record->severity_text, + strlen(log_record->severity_text))); + } + + if (ret == 0 && log_record->n_attributes > 0) { + ret = otel_cfl_insert_kvlist(builder, otlp, "attributes", 10, + otel_cfl_kvarray(builder, log_record->attributes, + log_record->n_attributes)); + } + + if (ret == 0 && log_record->dropped_attributes_count > 0) { + ret = otel_cfl_insert(builder, otlp, "dropped_attributes_count", 24, + otel_cfl_uint(builder, log_record->dropped_attributes_count)); + } + + if (ret == 0 && log_record->trace_id.len > 0) { + ret = otel_cfl_insert(builder, otlp, "trace_id", 8, + otel_cfl_bytes(builder, log_record->trace_id)); + } + + if (ret == 0 && log_record->span_id.len > 0) { + ret = otel_cfl_insert(builder, otlp, "span_id", 7, + otel_cfl_bytes(builder, log_record->span_id)); + } + + if (ret == 0) { + ret = otel_cfl_insert(builder, otlp, "trace_flags", 11, + otel_cfl_uint(builder, log_record->flags & 0xff)); + } + + if (ret == 0 && log_record->event_name != NULL && strlen(log_record->event_name) > 0) { + ret = otel_cfl_insert(builder, otlp, "event_name", 10, + otel_cfl_string(builder, log_record->event_name, + strlen(log_record->event_name))); + } + + if (ret != 0) { + goto error; + } + + ret = otel_cfl_insert_kvlist(builder, metadata, builder->options->metadata_key, + strlen(builder->options->metadata_key), otlp); + otlp = NULL; + if (ret != 0) { + goto error; + } + + return metadata; + +error: + if (otlp != NULL) { + cfl_kvlist_destroy(otlp); + } + if (metadata != NULL) { + cfl_kvlist_destroy(metadata); + } + return NULL; +} + +static struct cfl_kvlist *otel_cfl_body(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Common__V1__AnyValue *body) +{ + const char *logs_body_key; + struct cfl_kvlist *kvlist; + + if (builder->options->body_key == NULL && body != NULL && + body->value_case == OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_KVLIST_VALUE) { + return otel_cfl_kvarray(builder, body->kvlist_value->values, + body->kvlist_value->n_values); + } + + logs_body_key = builder->options->body_key; + if (logs_body_key == NULL) { + logs_body_key = "log"; + } + + kvlist = otel_cfl_kvlist_create_compact(builder, 1); + if (kvlist == NULL) { + return NULL; + } + + if (otel_cfl_insert(builder, kvlist, logs_body_key, strlen(logs_body_key), + otel_cfl_any_value(builder, body)) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + + return kvlist; +} + +static struct cfl_kvlist *otel_cfl_group_metadata(struct otel_cfl_builder *builder, + int resource_index, int scope_index) +{ + struct cfl_kvlist *kvlist; + + kvlist = otel_cfl_kvlist_create(builder); + if (kvlist == NULL) { + return NULL; + } + + if (otel_cfl_insert(builder, kvlist, "schema", 6, otel_cfl_string(builder, "otlp", 4)) != 0 || + otel_cfl_insert(builder, kvlist, "resource_id", 11, + otel_cfl_int(builder, resource_index)) != 0 || + otel_cfl_insert(builder, kvlist, "scope_id", 8, + otel_cfl_int(builder, scope_index)) != 0) { + cfl_kvlist_destroy(kvlist); + return NULL; + } + + return kvlist; +} + +static struct cfl_kvlist *otel_cfl_group_body( + struct otel_cfl_builder *builder, + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log, + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log) +{ + int ret = 0; + struct cfl_kvlist *body; + struct cfl_kvlist *resource_kv; + struct cfl_kvlist *scope_kv; + Opentelemetry__Proto__Resource__V1__Resource *resource; + Opentelemetry__Proto__Common__V1__InstrumentationScope *scope; + + body = otel_cfl_kvlist_create(builder); + resource_kv = otel_cfl_kvlist_create(builder); + scope_kv = otel_cfl_kvlist_create(builder); + if (body == NULL || resource_kv == NULL || scope_kv == NULL) { + goto error; + } + + resource = resource_log->resource; + if (resource) { + /* look for OTel resource attributes */ + if (resource->n_attributes > 0 && resource->attributes) { + ret = otel_cfl_insert_kvlist(builder, resource_kv, "attributes", 10, + otel_cfl_kvarray(builder, resource->attributes, + resource->n_attributes)); + } + + if (ret == 0 && resource->dropped_attributes_count > 0) { + ret = otel_cfl_insert(builder, resource_kv, "dropped_attributes_count", 24, + otel_cfl_uint(builder, resource->dropped_attributes_count)); + } + + if (ret == 0 && resource_log->schema_url) { + ret = otel_cfl_insert(builder, resource_kv, "schema_url", 10, + otel_cfl_string(builder, resource_log->schema_url, + strlen(resource_log->schema_url))); + } + } + + scope = scope_log->scope; + if (ret == 0 && scope_log->schema_url && strlen(scope_log->schema_url) > 0) { + ret = otel_cfl_insert(builder, scope_kv, "schema_url", 10, + otel_cfl_string(builder, scope_log->schema_url, + strlen(scope_log->schema_url))); + } + + if (scope) { + if (ret == 0 && scope->name && strlen(scope->name) > 0) { + ret = otel_cfl_insert(builder, scope_kv, "name", 4, + otel_cfl_string(builder, scope->name, strlen(scope->name))); + } + + if (ret == 0 && scope->version && strlen(scope->version) > 0) { + ret = otel_cfl_insert(builder, scope_kv, "version", 7, + otel_cfl_string(builder, scope->version, + strlen(scope->version))); + } + + if (ret == 0 && scope->n_attributes > 0 && scope->attributes) { + ret = otel_cfl_insert_kvlist(builder, scope_kv, "attributes", 10, + otel_cfl_kvarray(builder, scope->attributes, + scope->n_attributes)); + } + + if (ret == 0 && scope->dropped_attributes_count > 0) { + ret = otel_cfl_insert(builder, scope_kv, "dropped_attributes_count", 24, + otel_cfl_uint(builder, scope->dropped_attributes_count)); + } + } + + if (ret != 0) { + goto error; + } + + ret = otel_cfl_insert_kvlist(builder, body, "resource", 8, resource_kv); + resource_kv = NULL; + if (ret == 0) { + ret = otel_cfl_insert_kvlist(builder, body, "scope", 5, scope_kv); + scope_kv = NULL; + } + if (ret != 0) { + goto error; + } + + return body; + +error: + if (scope_kv != NULL) { + cfl_kvlist_destroy(scope_kv); + } + if (resource_kv != NULL) { + cfl_kvlist_destroy(resource_kv); + } + if (body != NULL) { + cfl_kvlist_destroy(body); + } + return NULL; +} + +/* Build the group start, records and group end of one scope logs entry */ +static int otel_cfl_scope_logs(struct otel_cfl_builder *builder, + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log, + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log, + int resource_index, int scope_index, + size_t *record_count) +{ + size_t index; + struct flb_time tm; + struct flb_mp_chunk_record *group; + struct flb_mp_chunk_record *record; + Opentelemetry__Proto__Logs__V1__LogRecord *log_record; + + flb_time_set(&tm, FLB_LOG_EVENT_GROUP_START, 0); + group = otel_cfl_record_add(builder, &tm); + if (group == NULL) { + return -1; + } + + group->cobj_metadata = otel_cfl_object(builder, + otel_cfl_group_metadata(builder, resource_index, scope_index)); + group->cobj_record = otel_cfl_object(builder, + otel_cfl_group_body(builder, resource_log, scope_log)); + if (group->cobj_metadata == NULL || group->cobj_record == NULL) { + return -1; + } + + /* the group start owns the group content, like a decoded chunk */ + group->cobj_group_metadata = group->cobj_metadata; + group->cobj_group_attributes = group->cobj_record; + group->owns_group_metadata = FLB_TRUE; + group->owns_group_attributes = FLB_TRUE; + + for (index = 0; index < scope_log->n_log_records; index++) { + log_record = scope_log->log_records[index]; + + if (log_record->time_unix_nano > 0) { + if (flb_time_from_uint64(&tm, log_record->time_unix_nano) != 0) { + return -1; + } + } + else if (log_record->observed_time_unix_nano > 0) { + if (flb_time_from_uint64(&tm, log_record->observed_time_unix_nano) != 0) { + return -1; + } + } + else { + flb_time_get(&tm); + } + + record = otel_cfl_record_add(builder, &tm); + if (record == NULL) { + return -1; + } + + record->cobj_metadata = otel_cfl_object(builder, + otel_cfl_v1_metadata(builder, log_record)); + if (record->cobj_metadata == NULL) { + flb_error("[opentelemetry] failed to convert log record"); + return -1; + } + + record->cobj_record = otel_cfl_object(builder, + otel_cfl_body(builder, log_record->body)); + if (record->cobj_record == NULL) { + flb_error("[opentelemetry] failed to convert log record body"); + return -1; + } + + record->cobj_group_metadata = group->cobj_group_metadata; + record->cobj_group_attributes = group->cobj_group_attributes; + (*record_count)++; + } + + flb_time_set(&tm, FLB_LOG_EVENT_GROUP_END, 0); + record = otel_cfl_record_add(builder, &tm); + if (record == NULL) { + return -1; + } + + record->cobj_metadata = otel_cfl_object(builder, otel_cfl_kvlist_create(builder)); + record->cobj_record = otel_cfl_object(builder, otel_cfl_kvlist_create(builder)); + if (record->cobj_metadata == NULL || record->cobj_record == NULL) { + return -1; + } + record->cobj_group_metadata = group->cobj_group_metadata; + record->cobj_group_attributes = group->cobj_group_attributes; + + return 0; +} + +/* + * CFL records borrow strings and byte arrays from the unpacked protobuf + * request. Keep the request and its arena alive until the caller consumes the + * chunk synchronously. + */ +struct flb_opentelemetry_logs_protobuf_cfl { + struct flb_mp_chunk_cobj *chunk; + size_t encoded_size; + size_t record_count; + struct cfl_arena *protobuf_arena; + ProtobufCAllocator arena_allocator; + ProtobufCAllocator *protobuf_allocator; + Opentelemetry__Proto__Collector__Logs__V1__ExportLogsServiceRequest *input_logs; +}; + +int flb_opentelemetry_logs_protobuf_to_cfl( + const void *payload, size_t payload_size, + const struct flb_opentelemetry_logs_decode_options *options, + struct flb_opentelemetry_logs_protobuf_cfl **out_result) +{ + int ret; + size_t resource_logs_index; + size_t scope_log_index; + struct otel_cfl_builder builder; + struct flb_opentelemetry_logs_protobuf_cfl *result; + Opentelemetry__Proto__Logs__V1__ResourceLogs *resource_log; + Opentelemetry__Proto__Logs__V1__ScopeLogs *scope_log; + + if (options == NULL || options->metadata_key == NULL || out_result == NULL || + (payload == NULL && payload_size != 0)) { + return -1; + } + + *out_result = NULL; + result = flb_calloc(1, sizeof(struct flb_opentelemetry_logs_protobuf_cfl)); + if (result == NULL) { + flb_errno(); + return -1; + } + + builder.options = options; + builder.arena = NULL; + builder.size = 0; + builder.limit = options->output_limit; + builder.chunk = NULL; + ret = 0; + + result->input_logs = protobuf_logs_decode( + (uint8_t *) payload, payload_size, + &result->protobuf_arena, + &result->arena_allocator, + &result->protobuf_allocator); + if (result->input_logs == NULL) { + flb_warn("[opentelemetry] failed to unpack protobuf logs payload"); + ret = -1; + goto end; + } + /* + * The CFL tree is consumed and encoded synchronously before the decoded + * protobuf request is released, so both representations can safely share + * the request arena. This avoids one heap allocation per CFL container and + * scalar while preserving the normal allocator fallback for small payloads. + */ + builder.arena = result->protobuf_arena; + + if (result->input_logs->n_resource_logs > 0 && + result->input_logs->resource_logs == NULL) { + flb_warn("[opentelemetry] no resource logs found"); + ret = -1; + goto end; + } + + builder.chunk = flb_mp_chunk_cobj_create(NULL, NULL); + if (builder.chunk == NULL) { + ret = -1; + goto end; + } + builder.chunk->record_pool_enabled = FLB_TRUE; + + for (resource_logs_index = 0; + resource_logs_index < result->input_logs->n_resource_logs; + resource_logs_index++) { + resource_log = result->input_logs->resource_logs[resource_logs_index]; + if (resource_log == NULL) { + flb_warn("[opentelemetry] null resource logs entry found"); + ret = -1; + goto end; + } + + if (resource_log->n_scope_logs > 0 && resource_log->scope_logs == NULL) { + flb_warn("[opentelemetry] no scope logs found"); + ret = -1; + goto end; + } + + for (scope_log_index = 0; scope_log_index < resource_log->n_scope_logs; + scope_log_index++) { + scope_log = resource_log->scope_logs[scope_log_index]; + if (scope_log == NULL) { + flb_warn("[opentelemetry] null scope logs entry found"); + ret = -1; + goto end; + } + if (scope_log->n_log_records == 0) { + continue; + } + if (scope_log->log_records == NULL) { + flb_warn("[opentelemetry] no log records found"); + ret = -1; + goto end; + } + + ret = otel_cfl_scope_logs(&builder, resource_log, scope_log, + resource_logs_index, scope_log_index, + &result->record_count); + if (ret != 0) { + flb_error("[opentelemetry] marshalling error"); + goto end; + } + } + } + + result->encoded_size = builder.size; + result->chunk = builder.chunk; + builder.chunk = NULL; + *out_result = result; + +end: + if (builder.chunk != NULL) { + flb_mp_chunk_cobj_destroy(builder.chunk); + } + if (ret != 0) { + flb_opentelemetry_logs_protobuf_cfl_destroy(result); + return -1; + } + + return 0; +} + +struct flb_mp_chunk_cobj *flb_opentelemetry_logs_protobuf_cfl_take_chunk( + struct flb_opentelemetry_logs_protobuf_cfl *result) +{ + struct flb_mp_chunk_cobj *chunk; + + if (result == NULL) { + return NULL; + } + + chunk = result->chunk; + result->chunk = NULL; + + return chunk; +} + +size_t flb_opentelemetry_logs_protobuf_cfl_get_encoded_size( + struct flb_opentelemetry_logs_protobuf_cfl *result) +{ + if (result == NULL) { + return 0; + } + + return result->encoded_size; +} + +void flb_opentelemetry_logs_protobuf_cfl_destroy( + struct flb_opentelemetry_logs_protobuf_cfl *result) +{ + if (result == NULL) { + return; + } + + if (result->chunk != NULL) { + flb_mp_chunk_cobj_destroy(result->chunk); + } + if (result->input_logs != NULL) { + protobuf_logs_free(result->input_logs, result->protobuf_allocator); + } + cfl_arena_destroy(result->protobuf_arena); + flb_free(result); +} diff --git a/plugins/in_opentelemetry/opentelemetry_protobuf.c b/src/opentelemetry/flb_opentelemetry_protobuf.c similarity index 91% rename from plugins/in_opentelemetry/opentelemetry_protobuf.c rename to src/opentelemetry/flb_opentelemetry_protobuf.c index ff302ddc3b0..736df7819f4 100644 --- a/plugins/in_opentelemetry/opentelemetry_protobuf.c +++ b/src/opentelemetry/flb_opentelemetry_protobuf.c @@ -17,7 +17,10 @@ * limitations under the License. */ -#include "opentelemetry_protobuf.h" +#include + +/* Includes the export request and intermediate protobuf message wrappers. */ +#define OTEL_PROTOBUF_MAX_DEPTH 100 struct protobuf_frame { const ProtobufCMessageDescriptor *descriptor; @@ -52,8 +55,8 @@ static int read_varint(struct protobuf_frame *frame, uint64_t *value) * Strings, bytes, packed scalars and unknown fields are opaque. The explicit * stack bounds our own stack usage as well as the subsequent unpack operation. */ -int opentelemetry_protobuf_validate(const ProtobufCMessageDescriptor *descriptor, - const void *data, size_t size) +int flb_opentelemetry_protobuf_validate(const ProtobufCMessageDescriptor *descriptor, + const void *data, size_t size) { struct protobuf_frame frames[OTEL_PROTOBUF_MAX_DEPTH]; struct protobuf_frame *frame; diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit_processors.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit_processors.yaml new file mode 100644 index 00000000000..8faa9775f00 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit_processors.yaml @@ -0,0 +1,31 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + http_server.workers: 4 + http_server.ingress_queue_event_limit: 1024 + http_server.ingress_queue_byte_limit: 2000 + processors: + logs: + - name: content_modifier + context: attributes + action: upsert + key: "processed" + value: "yes" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py index bd368d88654..f6d7e688ff3 100644 --- a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py +++ b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py @@ -20,6 +20,7 @@ import time import base64 import gzip +import http.client from concurrent.futures import ThreadPoolExecutor import grpc import requests @@ -73,6 +74,8 @@ IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_CONFIG = "otlp_http1_cleartext_workers_tiny_ingress_queue.yaml" IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_CONFIG = \ "otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit.yaml" +IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_PROCESSORS_CONFIG = \ + "otlp_http1_cleartext_workers_tiny_ingress_queue_byte_limit_processors.yaml" IN_OPENTELEMETRY_OAUTH2_PROTOCOL_CONFIGS = { "http1_cleartext": "otlp_http1_cleartext_oauth2.yaml", @@ -520,7 +523,8 @@ def generated_limit_service(tmp_path, workers, limit="16k"): (32, 64, 1, 1, False), (64, 2, 1, 2, False), (1, 1, 512, 1, False), - (64, 1000, 0, 1, False), + # scopes without records produce nothing, the request is accepted + (64, 1000, 0, 1, True), ]) def test_in_opentelemetry_generated_limit(tmp_path, workers, encoding, attributes, scopes, records, resources, accepted): @@ -2145,8 +2149,16 @@ def test_in_opentelemetry_http_workers_respect_tiny_ingress_queue_limits(): ) == 0 -def test_in_opentelemetry_http_workers_respect_ingress_queue_byte_limit(): - service = Service(IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_CONFIG) +@pytest.mark.parametrize( + "config_file", + [ + IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_CONFIG, + IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_PROCESSORS_CONFIG, + ], + ids=["no_processors", "worker_processors"], +) +def test_in_opentelemetry_http_workers_respect_ingress_queue_byte_limit(config_file): + service = Service(config_file) small_payload = service.build_otel_payload("test_logs_001.in.json", "logs") small_request = ExportLogsServiceRequest() @@ -2157,6 +2169,7 @@ def test_in_opentelemetry_http_workers_respect_ingress_queue_byte_limit(): large_request.resource_logs.extend(small_request.resource_logs) large_payload = large_request.SerializeToString() + large_json_payload = json_format.MessageToJson(large_request).encode() assert len(small_payload) < len(large_payload) @@ -2174,6 +2187,12 @@ def test_in_opentelemetry_http_workers_respect_ingress_queue_byte_limit(): assert failure_response.status_code == 503 assert "deferred ingress queue is full" in failure_response.text + failure_response = service.send_raw_request( + "/v1/logs", large_json_payload, content_type="application/json" + ) + assert failure_response.status_code == 503 + assert "deferred ingress queue is full" in failure_response.text + metrics_text = service.service.wait_for_condition( lambda: ( metrics if maybe_read_prometheus_metric_value( @@ -2696,3 +2715,724 @@ def test_in_opentelemetry_http_snappy_invalid_checksum(frame_type): service.wait_for_signal_count("logs", 1) finally: service.stop() + + +# Processor chains used to compare the msgpack and the direct CFL logs paths. +# The direct path is only taken when the chain starts with a native processor, +# the filter first chain covers the fallback to the msgpack path. +# +# The msgpack reference runs the same chain behind a filter that keeps every +# record: a filter unit makes the input take the established msgpack path. +MSGPACK_REFERENCE_FILTER = {"name": "grep", "exclude": "msgpack_reference ^never$"} + +DIRECT_CFL_PROCESSOR_CHAINS = { + "content_modifier": [ + {"name": "content_modifier", "context": "otel_resource_attributes", + "action": "upsert", "key": "aaa", "value": "bbb"}, + {"name": "content_modifier", "context": "otel_resource_attributes", + "action": "delete", "key": "service.name"}, + {"name": "content_modifier", "context": "otel_scope_name", + "action": "upsert", "value": "new scope name"}, + {"name": "content_modifier", "context": "attributes", + "action": "upsert", "key": "added", "value": "yes"}, + ], + "sql_drop": [ + {"name": "sql", "query": "SELECT * FROM STREAM WHERE keep = 'yes';"}, + {"name": "content_modifier", "context": "otel_scope_version", + "action": "upsert", "value": "3.1.0"}, + ], + "native_then_filter": [ + {"name": "content_modifier", "context": "otel_scope_attributes", + "action": "upsert", "key": "scope_new", "value": "123"}, + {"name": "grep", "regex": "keep ^yes$"}, + ], + "filter_first": [ + {"name": "grep", "regex": "keep ^yes$"}, + {"name": "content_modifier", "context": "otel_resource_attributes", + "action": "upsert", "key": "aaa", "value": "bbb"}, + ], + "hash_once": [ + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + ], + "rename": [ + {"name": "content_modifier", "context": "body", + "action": "rename", "key": "marker", "value": "renamed_marker"}, + ], + "hash_then_sql": [ + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + {"name": "sql", "query": "SELECT * FROM STREAM WHERE keep = 'yes';"}, + ], + "conditional_safe": [ + {"name": "content_modifier", "context": "attributes", + "action": "upsert", "key": "condition_matched", "value": "yes", + "condition": {"op": "and", "rules": [ + {"field": "$keep", "op": "eq", "value": "yes"} + ]}}, + {"name": "content_modifier", "context": "attributes", + "action": "upsert", "key": "always", "value": "yes"}, + ], + "hash_then_filter": [ + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + {"name": "grep", "regex": "keep ^yes$"}, + ], + "hash_then_unflagged": [ + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + {"name": "opentelemetry_envelope"}, + ], + "unflagged_then_hash": [ + {"name": "opentelemetry_envelope"}, + {"name": "content_modifier", "context": "attributes", + "action": "hash", "key": "marker"}, + ], +} + + +def add_any_value_attributes(attributes): + attributes.add(key="str").value.string_value = "value" + attributes.add(key="int_negative").value.int_value = -(2**40) + attributes.add(key="int_positive").value.int_value = 2**62 + attributes.add(key="double").value.double_value = 3.25 + attributes.add(key="bool").value.bool_value = True + attributes.add(key="bytes").value.bytes_value = b"\x00\x01binary\xff" + attributes.add(key="empty") + array = attributes.add(key="array").value.array_value + array.values.add().string_value = "one" + array.values.add().int_value = 2 + array.values.add() + nested = array.values.add().kvlist_value + nested.values.add(key="inner").value.bool_value = False + kvlist = attributes.add(key="kvlist").value.kvlist_value + kvlist.values.add(key="name").value.string_value = "nested" + kvlist.values.add(key="list").value.array_value.values.add().double_value = -1.5 + + +def add_kvlist_body(record, keep, marker): + body = record.body.kvlist_value + body.values.add(key="keep").value.string_value = keep + body.values.add(key="marker").value.string_value = marker + body.values.add(key="count").value.int_value = -5 + add_any_value_attributes(body.values.add(key="nested").value.kvlist_value.values) + + +def build_direct_cfl_logs_payload(): + request = ExportLogsServiceRequest() + timestamp = 1789516800123456789 + + resource_logs = request.resource_logs.add(schema_url="https://example.com/resource") + resource_logs.resource.attributes.add(key="service.name").value.string_value = "direct" + add_any_value_attributes(resource_logs.resource.attributes) + resource_logs.resource.dropped_attributes_count = 2 + + scope_logs = resource_logs.scope_logs.add(schema_url="https://example.com/scope") + scope_logs.scope.name = "direct-cfl" + scope_logs.scope.version = "1.0.0" + scope_logs.scope.dropped_attributes_count = 1 + add_any_value_attributes(scope_logs.scope.attributes) + + record = scope_logs.log_records.add(time_unix_nano=timestamp, + observed_time_unix_nano=timestamp + 1, + severity_number=9, severity_text="INFO", + dropped_attributes_count=3, flags=0x101, + trace_id=bytes(range(16)), span_id=bytes(range(8))) + add_kvlist_body(record, "yes", "kvlist") + add_any_value_attributes(record.attributes) + + scope_logs.log_records.add(time_unix_nano=timestamp + 2).body.string_value = "plain" + scope_logs.log_records.add(observed_time_unix_nano=timestamp + 3).body.int_value = -7 + scope_logs.log_records.add(time_unix_nano=timestamp + 4, + severity_number=24).body.bytes_value = b"\xde\xad" + array = scope_logs.log_records.add(time_unix_nano=timestamp + 5).body.array_value + array.values.add().double_value = 0.5 + array.values.add().bool_value = False + scope_logs.log_records.add(time_unix_nano=timestamp + 6, severity_number=99) + + # scopes without records produce no group + resource_logs.scope_logs.add().scope.name = "empty" + + # resources and scopes without content + second = request.resource_logs.add() + second_scope = second.scope_logs.add() + add_kvlist_body(second_scope.log_records.add(time_unix_nano=timestamp + 7), "yes", "bare") + add_kvlist_body(second_scope.log_records.add(time_unix_nano=timestamp + 8), "no", "bare") + second.scope_logs.add(schema_url="https://example.com/only-schema") + add_kvlist_body(second.scope_logs[1].log_records.add(time_unix_nano=timestamp + 9), + "yes", "schema-only") + + return request + + +def build_direct_cfl_sentinel_payload(keep="yes"): + request = ExportLogsServiceRequest() + record = request.resource_logs.add().scope_logs.add().log_records.add( + time_unix_nano=1789516800000000000) + add_kvlist_body(record, keep, "sentinel") + return request + + +def direct_cfl_service(tmp_path, direct, processors, *, body_key=None, workers=1, + logs_max_size=None, threaded=False, conditional_routes=False): + name = f"direct-cfl-{'on' if direct else 'off'}.yaml" + if not direct: + processors = [MSGPACK_REFERENCE_FILTER] + (processors or []) + options = [f"http_server.workers: {workers}"] + if body_key: + options.append(f"logs_body_key: {body_key}") + if logs_max_size: + options.append(f"logs_max_size: {logs_max_size}") + if threaded: + options.append("threaded: true") + option_lines = "\n".join(f" {option}" for option in options) + processor_lines = "" + if processors is not None: + processor_lines = f" processors: {json.dumps({'logs': processors})}\n" + route_lines = "" + output_match = " match: '*'" + if conditional_routes: + route_lines = """ routes: + logs: + - name: direct_cfl_default + condition: + default: true + to: + outputs: + - direct_cfl_output +""" + output_match = "" + config = tmp_path / name + config.write_text(f""" +service: + flush: 0.2 + grace: 1 + log_level: info + http_server: on + http_port: ${{FLUENT_BIT_HTTP_MONITORING_PORT}} +pipeline: + inputs: + - name: opentelemetry + port: ${{FLUENT_BIT_TEST_LISTENER_PORT}} + http2: on +{option_lines} +{processor_lines}{route_lines} outputs: + - name: opentelemetry + alias: direct_cfl_output +{output_match} + host: 127.0.0.1 + port: ${{TEST_SUITE_HTTP_PORT}} +""", encoding="utf-8") + return Service(str(config)) + + +def flatten_received_logs(requests_received): + records = [] + for request in requests_received: + for resource_logs in request.resource_logs: + resource = json_format.MessageToDict(resource_logs.resource) + for scope_logs in resource_logs.scope_logs: + scope = json_format.MessageToDict(scope_logs.scope) + for record in scope_logs.log_records: + records.append(json.dumps({ + "resource": resource, + "resource_schema_url": resource_logs.schema_url, + "scope": scope, + "scope_schema_url": scope_logs.schema_url, + "record": json_format.MessageToDict(record), + }, sort_keys=True)) + return sorted(records) + + +def is_sentinel(record): + return '"stringValue": "sentinel"' in record + + +def run_direct_cfl_case(tmp_path, direct, processors, transport, *, body_key=None, + workers=1, sentinel_keep="yes", threaded=False, + conditional_routes=False, encoding="protobuf"): + service = direct_cfl_service(tmp_path, direct, processors, body_key=body_key, + workers=workers, threaded=threaded, + conditional_routes=conditional_routes) + service.start() + try: + before = len(data_storage["logs"]) + payloads = [build_direct_cfl_logs_payload(), + build_direct_cfl_sentinel_payload(sentinel_keep)] + for payload in payloads: + if transport == "grpc": + service.send_grpc_request("logs", payload) + elif encoding == "json": + # protobuf's generic JSON formatter represents bytes as base64, + # while OTLP/JSON requires trace and span IDs in hexadecimal. + for resource_logs in payload.resource_logs: + for scope_logs in resource_logs.scope_logs: + for record in scope_logs.log_records: + record.trace_id = b"" + record.span_id = b"" + json_payload = json_format.MessageToJson(payload).encode("utf-8") + response = service.send_raw_request( + "/v1/logs", json_payload, content_type="application/json") + assert response.status_code == 201 + else: + response = service.send_raw_request("/v1/logs", payload.SerializeToString()) + assert response.status_code == 201 + + # The sentinel goes in its own request, after the payload being compared. + def received(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if any(is_sentinel(record) for record in records) else None + + return service.service.wait_for_condition(received, timeout=20, interval=0.1, + description="direct CFL sentinel") + finally: + service.stop() + + +@pytest.mark.parametrize("transport", ["http", "grpc"]) +@pytest.mark.parametrize("chain", sorted(DIRECT_CFL_PROCESSOR_CHAINS)) +def test_in_opentelemetry_logs_direct_cfl_parity(tmp_path, chain, transport): + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + msgpack_records = run_direct_cfl_case(tmp_path, False, processors, transport) + direct_records = run_direct_cfl_case(tmp_path, True, processors, transport) + + # more than the sentinel must reach the output for the comparison to matter + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.parametrize("chain", sorted(DIRECT_CFL_PROCESSOR_CHAINS)) +def test_in_opentelemetry_logs_direct_cfl_json_parity(tmp_path, chain): + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + msgpack_records = run_direct_cfl_case( + tmp_path, False, processors, "http", encoding="json") + direct_records = run_direct_cfl_case( + tmp_path, True, processors, "http", encoding="json") + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.parametrize("workers", [1, pytest.param( + 4, marks=pytest.mark.skipif(os.name == "nt", + reason="HTTP shared listener ports are unavailable on Windows"))]) +def test_in_opentelemetry_logs_direct_cfl_json_without_processors(tmp_path, workers): + # Without processors JSON uses the single-pass CFL decoder and is encoded + # once, the records must match the msgpack reference. + msgpack_records = run_direct_cfl_case( + tmp_path, False, None, "http", encoding="json", workers=workers) + direct_records = run_direct_cfl_case( + tmp_path, True, None, "http", encoding="json", workers=workers) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.parametrize("processors", [None, "content_modifier"]) +def test_in_opentelemetry_logs_direct_cfl_json_edge_parity(tmp_path, processors): + if processors is not None: + processors = DIRECT_CFL_PROCESSOR_CHAINS[processors] + payload = { + "resourceLogs": [{ + "resource": {"attributes": [ + {"key": "missing"}, + {"key": "unset", "value": {}}, + {"key": "nested", "value": { + "customValue": {"child": [1, True, None]} + }}, + ]}, + "scopeLogs": [{"logRecords": [ + {"timeUnixNano": "1789516800123456789", + "body": {"arrayValue": {"values": [ + {}, {"stringValue": "value"} + ]}}}, + {"timeUnixNano": "1789516800123456790", "body": {}}, + ]}], + }] + } + results = [] + + for direct in [False, True]: + service = direct_cfl_service(tmp_path, direct, processors) + try: + service.start() + before = len(data_storage["logs"]) + response = service.send_raw_request( + "/v1/logs", json.dumps(payload).encode("utf-8"), + content_type="application/json") + assert response.status_code == 201 + + def received_records(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if len(records) == 2 else None + + received = service.service.wait_for_condition( + received_records, + timeout=20, interval=0.1, description="direct CFL JSON edge records") + results.append(received) + finally: + service.stop() + + assert results[1] == results[0] + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +def test_in_opentelemetry_logs_direct_cfl_empty_request(tmp_path, encoding): + # A request without log records is valid and accepted on every path. + if encoding == "json": + payload = b'{"resourceLogs":[{"scopeLogs":[{"logRecords":[]}]}]}' + content_type = "application/json" + else: + request = ExportLogsServiceRequest() + request.resource_logs.add().scope_logs.add() + payload = request.SerializeToString() + content_type = "application/x-protobuf" + + for direct in [False, True]: + service = direct_cfl_service(tmp_path, direct, + DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"]) + try: + service.start() + response = service.send_raw_request("/v1/logs", payload, + content_type=content_type) + assert response.status_code == 201 + finally: + service.stop() + + +@pytest.mark.parametrize("processors", [None, "content_modifier"]) +def test_in_opentelemetry_logs_direct_cfl_json_unsupported_key(tmp_path, processors): + # CFL maps only hold string keys. A wrapped attribute key is unwrapped, + # a numeric one makes the request take the msgpack path. + if processors is not None: + processors = DIRECT_CFL_PROCESSOR_CHAINS[processors] + payload = {"resourceLogs": [{"scopeLogs": [{"logRecords": [ + {"timeUnixNano": "1789516800000000000", "body": {"stringValue": "wrapped"}, + "attributes": [{"key": {"stringValue": "name"}, "value": {"intValue": "1"}}]}, + {"timeUnixNano": "1789516800000000001", "body": {"stringValue": "numeric"}, + "attributes": [{"key": 5, "value": {"boolValue": True}}]}, + ]}]}]} + sentinel = build_direct_cfl_sentinel_payload().SerializeToString() + results = [] + + for direct in [False, True]: + service = direct_cfl_service(tmp_path, direct, processors) + try: + service.start() + before = len(data_storage["logs"]) + response = service.send_raw_request( + "/v1/logs", json.dumps(payload).encode("utf-8"), + content_type="application/json") + assert response.status_code == 201 + response = service.send_raw_request("/v1/logs", sentinel) + assert response.status_code == 201 + + def received(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if any(is_sentinel(record) for record in records) else None + + results.append(service.service.wait_for_condition( + received, timeout=20, interval=0.1, description="sentinel after key test")) + finally: + service.stop() + + assert results[1] == results[0] + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +def test_in_opentelemetry_logs_direct_cfl_body_key(tmp_path, encoding): + processors = DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"] + msgpack_records = run_direct_cfl_case(tmp_path, False, processors, "http", + body_key="message", encoding=encoding) + direct_records = run_direct_cfl_case(tmp_path, True, processors, "http", + body_key="message", encoding=encoding) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +def test_in_opentelemetry_http_workers_balance_keepalive_connections(tmp_path, encoding): + # Every accepted socket may be handed to another worker. Many concurrent + # keep-alive connections must all be served and every record delivered. + connections = 24 + requests_per_connection = 10 + service = direct_cfl_service(tmp_path, True, DIRECT_CFL_PROCESSOR_CHAINS["hash_once"], + workers=4, threaded=True) + service.start() + try: + before = len(data_storage["logs"]) + payload = build_direct_cfl_sentinel_payload("keep") + if encoding == "json": + body = json_format.MessageToJson(payload).encode("utf-8") + content_type = "application/json" + else: + body = payload.SerializeToString() + content_type = "application/x-protobuf" + + def client(): + connection = http.client.HTTPConnection("127.0.0.1", service.flb_listener_port, + timeout=10) + try: + for _ in range(requests_per_connection): + connection.request("POST", "/v1/logs", body, + {"Content-Type": content_type}) + response = connection.getresponse() + response.read() + assert response.status == 201 + finally: + connection.close() + + with ThreadPoolExecutor(max_workers=connections) as pool: + for future in [pool.submit(client) for _ in range(connections)]: + future.result() + + expected = connections * requests_per_connection + + def delivered(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if len(records) >= expected else None + + records = service.service.wait_for_condition(delivered, timeout=30, interval=0.2, + description="balanced connections") + assert len(records) == expected + finally: + service.stop() + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +def test_in_opentelemetry_logs_direct_cfl_workers(tmp_path): + # Hashing makes accidental processing in both the worker and collector + # observable: a double pass would hash the first digest again. + processors = DIRECT_CFL_PROCESSOR_CHAINS["hash_once"] + msgpack_records = run_direct_cfl_case(tmp_path, False, processors, "http", workers=4) + direct_records = run_direct_cfl_case(tmp_path, True, processors, "http", workers=4) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("conditional_routes", [False, True]) +def test_in_opentelemetry_logs_direct_cfl_threaded_workers(tmp_path, conditional_routes): + processors = DIRECT_CFL_PROCESSOR_CHAINS["hash_once"] + msgpack_records = run_direct_cfl_case( + tmp_path, False, processors, "http", workers=4, threaded=True, + conditional_routes=conditional_routes) + direct_records = run_direct_cfl_case( + tmp_path, True, processors, "http", workers=4, threaded=True, + conditional_routes=conditional_routes) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("conditional_routes", [False, True]) +def test_in_opentelemetry_logs_worker_processing(tmp_path, encoding, conditional_routes): + # Worker-safe chains run in the HTTP workers, the msgpack reference runs + # them in the input collector. + processors = DIRECT_CFL_PROCESSOR_CHAINS["hash_once"] + collector_records = run_direct_cfl_case( + tmp_path, False, processors, "http", workers=4, threaded=True, + conditional_routes=conditional_routes, encoding=encoding) + worker_records = run_direct_cfl_case( + tmp_path, True, processors, "http", workers=4, threaded=True, + conditional_routes=conditional_routes, encoding=encoding) + + assert len(collector_records) > 1 + assert worker_records == collector_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("chain", ["hash_then_filter", "hash_then_unflagged", + "unflagged_then_hash"]) +def test_in_opentelemetry_logs_worker_processing_fallback(tmp_path, chain): + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + collector_records = run_direct_cfl_case( + tmp_path, False, processors, "http", workers=4, threaded=True, + conditional_routes=True) + fallback_records = run_direct_cfl_case( + tmp_path, True, processors, "http", workers=4, threaded=True, + conditional_routes=True) + + assert len(collector_records) > 1 + assert fallback_records == collector_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("direct", [False, True]) +def test_in_opentelemetry_logs_direct_cfl_threaded_stress(tmp_path, direct): + request_count = 200 + processors = DIRECT_CFL_PROCESSOR_CHAINS["hash_once"] + service = direct_cfl_service( + tmp_path, direct, processors, workers=4, threaded=True, + conditional_routes=True) + payload = build_direct_cfl_sentinel_payload().SerializeToString() + + try: + service.start() + before = len(data_storage["logs"]) + + def send_request(_): + return service.send_raw_request("/v1/logs", payload).status_code + + with ThreadPoolExecutor(max_workers=16) as executor: + statuses = list(executor.map(send_request, range(request_count))) + + assert statuses == [201] * request_count + + def received_records(): + records = flatten_received_logs(data_storage["logs"][before:]) + return records if len(records) >= request_count else None + + records = service.service.wait_for_condition( + received_records, timeout=30, interval=0.1, + description="threaded direct-CFL stress records") + assert len(records) == request_count + finally: + service.stop() + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("chain", ["hash_then_filter", "filter_first", + "hash_then_unflagged", "unflagged_then_hash"]) +def test_in_opentelemetry_logs_direct_cfl_workers_fallback(tmp_path, chain): + # Hashing is intentionally non-idempotent. If an eligible prefix ran in a + # worker before the whole chain fell back, direct-on parity would fail. + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + msgpack_records = run_direct_cfl_case(tmp_path, False, processors, "http", workers=4) + direct_records = run_direct_cfl_case(tmp_path, True, processors, "http", workers=4) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.skipif(os.name == "nt", reason="HTTP shared listener ports are unavailable on Windows") +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("chain", ["content_modifier", "sql_drop", "hash_then_sql", + "conditional_safe"]) +def test_in_opentelemetry_logs_direct_cfl_threaded_worker_safe_chain(tmp_path, chain, + encoding): + processors = DIRECT_CFL_PROCESSOR_CHAINS[chain] + msgpack_records = run_direct_cfl_case( + tmp_path, False, processors, "http", workers=4, threaded=True, + encoding=encoding) + direct_records = run_direct_cfl_case( + tmp_path, True, processors, "http", workers=4, threaded=True, + encoding=encoding) + + assert len(msgpack_records) > 1 + assert direct_records == msgpack_records + + +@pytest.mark.parametrize("direct", [False, True]) +def test_in_opentelemetry_logs_direct_cfl_drop_all(tmp_path, direct): + # Every record of the payload is dropped, only the sentinel matches. + processors = [{"name": "sql", "query": "SELECT * FROM STREAM WHERE keep = 'last';"}] + records = run_direct_cfl_case(tmp_path, direct, processors, "http", sentinel_keep="last") + + assert len(records) == 1 + assert is_sentinel(records[0]) + + +@pytest.mark.parametrize("attributes,scopes,records,resources,accepted", [ + (1, 32, 1, 1, True), + (64, 1, 1, 1, True), + (64, 32, 1, 1, False), + (1, 1, 512, 1, False), + # scopes without records produce nothing, the request is accepted + (64, 1000, 0, 1, True), +]) +def test_in_opentelemetry_logs_direct_cfl_generated_limit(tmp_path, attributes, scopes, + records, resources, accepted): + processors = DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"] + service = direct_cfl_service(tmp_path, True, processors, logs_max_size="16k") + request = generated_limit_payload(attributes, scopes, records, resources) + try: + service.start() + response = service.send_raw_request("/v1/logs", request.SerializeToString()) + assert response.status_code == (201 if accepted else 400) + + # A rejected request never appends a prefix of its records. + small = generated_limit_payload(1, 1) + small.resource_logs[0].scope_logs[0].log_records[0].body.string_value = "recovery" + response = service.send_raw_request("/v1/logs", small.SerializeToString()) + assert response.status_code == 201 + + def received_records(): + return [record for request in data_storage["logs"] + for resource in request.resource_logs + for scope in resource.scope_logs for record in scope.log_records] + + expected = scopes * records * resources if accepted else 0 + service.service.wait_for_condition( + lambda: len(received_records()) >= expected + 1, + timeout=20, interval=0.1, description="recovery after direct CFL byte limit") + actual = received_records() + assert sum(record.body.string_value == "recovery" for record in actual) == 1 + assert sum(record.body.string_value == "generated-limit" for record in actual) == expected + finally: + service.stop() + + +@pytest.mark.parametrize("encoding", ["protobuf", "json"]) +@pytest.mark.parametrize("limit", [512, 1024, 2048, 4096, 8192, 16384]) +def test_in_opentelemetry_logs_direct_cfl_limit_parity(tmp_path, limit, encoding): + processors = DIRECT_CFL_PROCESSOR_CHAINS["content_modifier"] + request = build_direct_cfl_logs_payload() + if encoding == "json": + for resource_logs in request.resource_logs: + for scope_logs in resource_logs.scope_logs: + for record in scope_logs.log_records: + record.trace_id = b"" + record.span_id = b"" + payload = json_format.MessageToJson(request).encode("utf-8") + content_type = "application/json" + else: + payload = request.SerializeToString() + content_type = "application/x-protobuf" + statuses = [] + + for direct in [False, True]: + service = direct_cfl_service(tmp_path, direct, processors, + logs_max_size=str(limit)) + try: + service.start() + response = service.send_raw_request( + "/v1/logs", payload, content_type=content_type) + statuses.append(response.status_code) + finally: + service.stop() + + assert statuses[1] == statuses[0] + + +@pytest.mark.parametrize( + "config_file", + [ + "otlp_http1_cleartext_workers.yaml", + "otlp_http1_cleartext.yaml", + IN_OPENTELEMETRY_TINY_INGRESS_QUEUE_BYTE_LIMIT_PROCESSORS_CONFIG, + ], + ids=["no_processors", "processors", "worker_processors"], +) +def test_in_opentelemetry_json_logs_replace_invalid_unicode(config_file): + service = Service(config_file) + payload = ( + b'{"resourceLogs":[{"scopeLogs":[{"logRecords":[{"body":' + b'{"stringValue":"bad \\uD800 surrogate"}}]}]}]}' + ) + + service.start() + try: + logs_before = len(data_storage["logs"]) + response = service.send_raw_request( + "/v1/logs", payload, content_type="application/json" + ) + assert response.status_code == 201 + service.wait_for_signal_count("logs", logs_before + 1, timeout=10) + finally: + service.stop() diff --git a/tests/integration/scenarios/out_stdout/config/out_stdout_processor_workers.yaml b/tests/integration/scenarios/out_stdout/config/out_stdout_processor_workers.yaml new file mode 100644 index 00000000000..1f055be9ae0 --- /dev/null +++ b/tests/integration/scenarios/out_stdout/config/out_stdout_processor_workers.yaml @@ -0,0 +1,48 @@ +service: + flush: 0.1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: output_single + dummy: '{"message":"output worker single"}' + samples: 1 + + - name: dummy + tag: output_chain + dummy: '{"message":"output worker chain"}' + samples: 1 + + outputs: + - name: stdout + match: output_single + workers: 4 + processors: + logs: + - name: content_modifier + context: body + action: insert + key: native_one + value: yes + + - name: stdout + match: output_chain + workers: 4 + processors: + logs: + - name: content_modifier + context: body + action: insert + key: native_one + value: yes + - name: modify + add: filter_middle yes + - name: content_modifier + context: body + action: insert + key: native_two + value: yes diff --git a/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py b/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py index 33d847019a7..81262c11570 100644 --- a/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py +++ b/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py @@ -201,6 +201,27 @@ def test_out_stdout_logs_accepts_otlp_json_ingestion(): assert "This is another example log message." in log_text +def test_out_stdout_processor_chains_with_four_output_workers(): + service = Service("out_stdout_processor_workers.yaml") + service.start() + service.wait_for_log_contains("output worker single") + service.wait_for_log_contains("output worker chain") + log_text = service.read_log() + service.stop() + + # Concurrent stdout workers can interleave writes at byte granularity, so + # validate the complete captured stream rather than individual lines. + assert "] output_single:" in log_text + assert "] output_chain:" in log_text + assert log_text.count('"native_one"=>"yes"') == 2 + assert '"filter_middle"=>"yes"' in log_text + assert log_text.count('"native_two"=>"yes"') == 1 + for output_index in (0, 1): + for worker_index in range(4): + assert (f"[output:stdout:stdout.{output_index}] worker " + f"#{worker_index} started") in log_text + + def test_out_stdout_preserves_post_2038_otlp_timestamp(): service = Service("out_stdout_otel.yaml") service.start() diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 0b358e9bf2b..4e60f79673c 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -46,6 +46,7 @@ set(UNIT_TESTS_FILES ring_buffer.c regex.c scheduler.c + json.c parser_json.c parser_ltsv.c parser_regex.c diff --git a/tests/internal/data/opentelemetry/README.md b/tests/internal/data/opentelemetry/README.md index 656a34f073f..fa3005e7123 100644 --- a/tests/internal/data/opentelemetry/README.md +++ b/tests/internal/data/opentelemetry/README.md @@ -163,4 +163,17 @@ The test framework automatically detects whether a test case uses the legacy or - If `expected` contains a `groups` field, it uses the extended format - Otherwise, it falls back to the legacy format -This ensures all existing test cases continue to work without modification. \ No newline at end of file +This ensures all existing test cases continue to work without modification. +## CFL Parity Corpora + +`logs_parity.jsonl` (one OTLP/JSON request per line) and `logs_parity.pb` +(OTLP protobuf requests, each prefixed by its little-endian 32-bit length) are +random, often invalid, logs requests. The tests decode every request with both +the CFL and the msgpack decoders and expect the same records, the same +rejections and the same `logs_max_size` boundary. + +Regenerate them with: + +```shell +tests/integration/.venv/bin/python tests/internal/data/opentelemetry/logs_parity_corpus.py +``` diff --git a/tests/internal/data/opentelemetry/logs_parity.jsonl b/tests/internal/data/opentelemetry/logs_parity.jsonl new file mode 100644 index 00000000000..f8bbfe7adc6 --- /dev/null +++ b/tests/internal/data/opentelemetry/logs_parity.jsonl @@ -0,0 +1,200 @@ +{"resourceLogs": [{"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "1.25"}, "schemaUrl": "1.25", "scopeLogs": [true]}, {"resource": "abc", "scopeLogs": [{"schemaUrl": "aGVsbG8=", "logRecords": []}]}]} +{"resourceLogs": [{"resource": null, "scopeLogs": [{"schemaUrl": 1.5, "logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": 5, "droppedAttributesCount": "1.25", "flags": 1.5, "eventName": -5, "body": 1}, {"observedTimeUnixNano": "x", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": null, "flags": "aGVsbG8=", "body": 9223372036854775808}, {"severityText": 9223372036854775808, "droppedAttributesCount": -0.0, "eventName": "aGVsbG8="}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": 9223372036854775808, "scopeLogs": [{"scope": {"name": 9223372036854775808, "version": "123", "attributes": ["123", {"key": "k", "value": null}, {"key": {"stringValue": "w"}, "value": {"kvlistValue": 9223372036854775808}}], "droppedAttributesCount": "1.25"}}, {"scope": {"name": false, "version": -5, "attributes": [], "droppedAttributesCount": "aGVsbG8="}, "logRecords": {}}]}, {"schemaUrl": -0.0, "scopeLogs": [{"scope": "abc", "logRecords": [{"severityNumber": "abc", "traceId": "0123456789abcdef0123456789abcdef", "body": {}}]}, {"logRecords": [{"timeUnixNano": 1.5, "severityNumber": -0.0, "body": 0}, {"timeUnixNano": "9x", "severityNumber": "1.25", "attributes": "a", "traceId": "zz", "spanId": "0123456789abcdef", "flags": false, "eventName": null}, {"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": 5, "attributes": "-9", "droppedAttributesCount": 9223372036854775808, "spanId": "0123456789abcdef", "flags": "123"}]}]}]} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "a"}, "scopeLogs": [{"scope": {"name": -0.0, "version": "123", "attributes": [{"value": {"intValue": 9223372036854775808}}, {"key": null, "value": {"boolValue": "a"}}], "droppedAttributesCount": -0.0}, "logRecords": {}}, {"logRecords": [{"severityText": 1, "attributes": {}, "body": 9223372036854775808}, {"severityText": 9223372036854775808, "attributes": [], "droppedAttributesCount": false, "flags": -0.0, "eventName": 9223372036854775808, "body": {}}, {"observedTimeUnixNano": "", "severityNumber": -5, "severityText": "aGVsbG8=", "droppedAttributesCount": 1, "traceId": 5, "spanId": "0123", "flags": "s", "body": "1.25"}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": {"name": 1.5, "version": "-9", "attributes": [], "droppedAttributesCount": "abc"}, "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": []}, {"resource": -0.0, "scopeLogs": [{"scope": {}, "logRecords": [{"severityNumber": "1.25", "attributes": {}, "spanId": "0123456789abcdef", "body": 9223372036854775808}, -5, {"timeUnixNano": "", "attributes": [{"key": "", "value": false}, {"key": 5}, {"key": {"intValue": 1}, "value": [{"str": "123"}]}], "traceId": 5, "spanId": "", "body": {"b": [{"a": []}, {"str": {"bytesValue": 1}}]}}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"timeUnixNano": "1640995200000000000", "attributes": "", "droppedAttributesCount": false, "traceId": "0123456789abcdef0123456789abcdef", "flags": 0}]}, {"logRecords": {}}]}, "abc"]} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"scope": {}, "logRecords": [{"severityNumber": "aGVsbG8=", "severityText": -0.0, "droppedAttributesCount": "123", "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123", "body": "123"}, {"severityNumber": "-9", "attributes": {}, "droppedAttributesCount": true, "traceId": "", "flags": "a", "eventName": 1.5, "body": null}]}, {"scope": {"name": -0.0, "version": "-9", "attributes": [{"value": {"bytesValue": "abc"}}, {"key": null, "value": {"kvlistValue": {"values": [{"key": "k"}]}}}], "droppedAttributesCount": -5}, "logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": "x", "severityNumber": "aGVsbG8=", "droppedAttributesCount": null, "traceId": "0123456789abcdef0123456789abcdef", "flags": 9223372036854775808, "eventName": false}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "x", "severityNumber": 1, "attributes": false, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "", "flags": false}]}, {"scope": "aGVsbG8=", "logRecords": [{"severityText": 9223372036854775808, "droppedAttributesCount": -5, "traceId": "", "body": {"boolValue": "-9"}}, {"timeUnixNano": -1, "severityText": 1.5, "droppedAttributesCount": true, "traceId": "zz"}]}]}]} +{"resourceLogs": [{"schemaUrl": "abc", "scopeLogs": [{"scope": {"name": 1, "version": 1, "attributes": [0], "droppedAttributesCount": "-9"}, "logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "", "severityText": "1.25", "flags": "-9", "eventName": 1.5}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": 9223372036854775808, "logRecords": [{"droppedAttributesCount": "abc", "traceId": "0123456789abcdef0123456789abcdef", "spanId": ""}]}, {"scope": {}, "schemaUrl": "", "logRecords": []}, {"scope": 1, "logRecords": {}}]}]} +{"resourceLogs": [{"scopeLogs": [1.5, {"scope": {"name": "-9", "version": 1.5, "attributes": [], "droppedAttributesCount": "aGVsbG8="}, "logRecords": [{"severityText": -0.0, "droppedAttributesCount": "-9", "spanId": "0123", "eventName": "a"}, {"severityText": 9223372036854775808, "attributes": {}, "traceId": "", "spanId": null, "eventName": -0.0, "body": {"b": {"doubleValue": "a"}, "c": {"boolValue": false}}}]}, {"logRecords": -0.0}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": true}]} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": "s", "logRecords": [{"observedTimeUnixNano": "x", "flags": 1.5, "eventName": "123"}, {"timeUnixNano": "1640995200000000000", "severityNumber": null, "droppedAttributesCount": 1.5, "flags": 1.5, "body": "1.25"}]}, {"scope": {"name": -5, "version": -0.0, "attributes": {}, "droppedAttributesCount": "1.25"}}, {"logRecords": [{"timeUnixNano": "99999999999999999999999", "severityNumber": "123", "severityText": "1.25", "attributes": {}, "flags": ""}, {"timeUnixNano": "99999999999999999999999", "severityText": -5, "traceId": "zz", "eventName": "abc"}, {"timeUnixNano": "", "severityText": "-9", "traceId": ""}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": -0.0}, "schemaUrl": 9223372036854775808, "scopeLogs": [{"logRecords": [{"timeUnixNano": "1640995200000000000", "attributes": [], "traceId": "0123456789abcdef0123456789abcdef", "spanId": null, "eventName": null}]}, {"logRecords": [{"timeUnixNano": "", "traceId": 5, "flags": "a", "eventName": "123", "body": {"": 0}}, {"timeUnixNano": -1, "spanId": "0123"}]}, {"scope": {}, "schemaUrl": "123", "logRecords": [{"timeUnixNano": 1.5, "severityNumber": 0, "spanId": "", "eventName": "abc"}, {"observedTimeUnixNano": 5, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123456789abcdef", "eventName": "-9", "body": {"bytesValue": true}}]}]}, {"scopeLogs": []}]} +{"resourceLogs": [{"schemaUrl": "s", "scopeLogs": [{"logRecords": [{"timeUnixNano": "9x", "severityText": 0, "attributes": {}, "spanId": "0123", "body": "a"}, {"timeUnixNano": "9x", "severityText": -5, "traceId": 5, "flags": "s", "eventName": 1.5}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": 5}]}]}, {"resource": {"attributes": [], "droppedAttributesCount": "a"}, "scopeLogs": [{"logRecords": [{"severityNumber": "123", "severityText": false, "traceId": "zz"}, {"timeUnixNano": 1640995200000000000, "severityNumber": 9223372036854775808, "severityText": true, "traceId": "", "flags": null, "eventName": null}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"logRecords": []}]}]} +{"resourceLogs": [{"resource": {"attributes": [{"key": {"intValue": 1}, "value": {"bytesValue": "s"}}, {"key": "", "value": {}}, {"key": null, "value": {"kvlistValue": true}}], "droppedAttributesCount": 1.5}, "schemaUrl": -5, "scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": "", "observedTimeUnixNano": "x", "severityNumber": "", "severityText": "123", "attributes": {}, "flags": true, "body": -0.0}]}, 0, {"scope": "1.25", "logRecords": [{"timeUnixNano": "9x", "severityText": -5, "spanId": "", "flags": "123", "eventName": 9223372036854775808}]}]}, {"resource": {"attributes": [{"key": {"stringValue": "w"}, "value": {"intValue": 1}}], "droppedAttributesCount": 1.5}, "scopeLogs": [{"logRecords": []}]}]} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": ""}, "scopeLogs": [{"scope": {}, "schemaUrl": "s", "logRecords": [{"severityText": 1.5, "attributes": {}, "traceId": "0123456789abcdef0123456789abcdef"}, {"observedTimeUnixNano": 5, "severityNumber": "123", "severityText": "123", "attributes": {}, "eventName": "abc"}]}, {"logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "x", "severityText": -0.0, "droppedAttributesCount": -0.0, "spanId": null, "flags": 0}, {"severityText": "a", "spanId": ""}, {"timeUnixNano": -1, "severityNumber": true, "severityText": true, "attributes": {}, "droppedAttributesCount": 1, "traceId": ""}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": {"name": "a", "version": "123", "attributes": [], "droppedAttributesCount": "-9"}, "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"timeUnixNano": 1.5, "severityNumber": 1.5, "droppedAttributesCount": 1, "spanId": null, "flags": "a", "body": 9223372036854775808}, {"observedTimeUnixNano": 5, "severityText": 1, "droppedAttributesCount": "s", "eventName": "-9", "body": {}}]}, {"scope": {}, "logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "severityNumber": "s", "droppedAttributesCount": "abc", "spanId": "0123", "body": {"int": 1}}, {"timeUnixNano": "9x", "severityNumber": "", "spanId": "0123", "flags": null, "body": {}}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": null}, "schemaUrl": "-9", "scopeLogs": [{"logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "", "severityNumber": "123", "severityText": "1.25", "droppedAttributesCount": "aGVsbG8=", "traceId": "0123456789abcdef0123456789abcdeg"}]}]}, {"resource": -5, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": {"intValue": 1}, "value": {"kvlistValue": 0}}, {"value": {"intValue": 0}}, {"key": null, "value": {"arrayValue": "-9"}}], "droppedAttributesCount": 1.5}, "schemaUrl": "a", "scopeLogs": []}, {"resource": {"attributes": [{"key": "k", "value": {"intValue": 1}}, 1.5], "droppedAttributesCount": -0.0}, "schemaUrl": false, "scopeLogs": []}]} +{"resourceLogs": [{"schemaUrl": "s", "scopeLogs": [{}, {"logRecords": [{"timeUnixNano": -1, "droppedAttributesCount": "a", "spanId": null}]}]}, {"scopeLogs": [{}, {"logRecords": [{"timeUnixNano": "9x", "severityText": false, "attributes": {}, "droppedAttributesCount": 1, "spanId": "0123", "flags": "a"}, {"timeUnixNano": -1, "severityNumber": "aGVsbG8=", "attributes": [{"key": "k2", "value": {"arrayValue": [9223372036854775808, -0.0, -5]}}], "droppedAttributesCount": "1.25", "spanId": "0123", "body": ""}, {"timeUnixNano": 1.5, "observedTimeUnixNano": "5", "severityNumber": "abc", "droppedAttributesCount": "aGVsbG8=", "traceId": "", "spanId": null, "flags": null}]}, {"logRecords": []}]}]} +{"resourceLogs": [{"schemaUrl": false, "scopeLogs": [{"scope": {}, "schemaUrl": true, "logRecords": [{"severityNumber": "s", "severityText": true, "droppedAttributesCount": false, "traceId": "0123456789abcdef0123456789abcdeg", "flags": 1.5, "body": false}, {"severityNumber": 1.5, "droppedAttributesCount": 9223372036854775808, "spanId": "0123456789abcdef", "body": true}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "5", "severityNumber": 1, "attributes": 1.5, "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "", "flags": 1, "body": false}]}, {"logRecords": {}}, {"logRecords": -5}]}, {"resource": {"attributes": [{"key": "", "value": {"int": "s"}}, {"key": 5, "value": {"intValue": "-9"}}, {}], "droppedAttributesCount": 0}, "scopeLogs": [{"schemaUrl": -5, "logRecords": []}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": 9223372036854775808, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": -5, "scopeLogs": [{"scope": 1, "logRecords": [{"timeUnixNano": "", "severityText": null}]}]}, {"schemaUrl": "123", "scopeLogs": [{"logRecords": [{"droppedAttributesCount": "a", "traceId": "0123456789abcdef0123456789abcdeg", "eventName": -0.0}, {"timeUnixNano": 1.5, "severityNumber": "aGVsbG8=", "traceId": 5}]}, {"logRecords": [{"observedTimeUnixNano": "", "traceId": "zz", "flags": 9223372036854775808, "eventName": "-9"}, {"timeUnixNano": "1640995200000000000", "severityNumber": "abc", "severityText": 1.5, "attributes": [], "droppedAttributesCount": "s", "spanId": "0123456789abcdef", "flags": "-9", "eventName": "s"}, {"observedTimeUnixNano": "x", "severityNumber": null, "severityText": -5, "attributes": {}, "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123", "eventName": "-9"}]}, {"logRecords": 0}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": []}, {"schemaUrl": "a", "scopeLogs": [{"schemaUrl": null, "logRecords": []}, {}]}]} +{"resourceLogs": [{"resource": "aGVsbG8=", "schemaUrl": "abc", "scopeLogs": [{"logRecords": [{"severityNumber": false, "severityText": "123", "droppedAttributesCount": "", "spanId": null, "eventName": "", "body": {"a": {}, "c": {"kvlistValue": {}}}}, {"droppedAttributesCount": "-9", "spanId": ""}, true]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": false}, "scopeLogs": [{"scope": {"name": true, "version": "123", "attributes": [], "droppedAttributesCount": 9223372036854775808}, "logRecords": []}, {"logRecords": []}, {"logRecords": [{"observedTimeUnixNano": 5, "severityNumber": true, "droppedAttributesCount": null, "flags": 0, "body": "aGVsbG8="}]}]}]} +{"resourceLogs": [{"resource": true, "schemaUrl": 1.5, "scopeLogs": true}]} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": 0, "logRecords": [{"observedTimeUnixNano": "", "severityText": null, "body": {"": 9223372036854775808}}]}, {"scope": {}, "logRecords": [{"timeUnixNano": "9x", "traceId": "0123456789abcdef0123456789abcdef", "spanId": "", "flags": "123", "body": {}}]}, {"schemaUrl": "a", "logRecords": [{"observedTimeUnixNano": "5", "severityText": null, "droppedAttributesCount": 0, "spanId": null, "eventName": ""}]}]}, {"schemaUrl": -0.0, "scopeLogs": [{"logRecords": []}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": -0.0, "schemaUrl": 0, "scopeLogs": [{"scope": {"name": 0, "version": "abc", "attributes": [], "droppedAttributesCount": false}, "schemaUrl": null, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": 5, "severityText": "123", "attributes": [], "droppedAttributesCount": "-9", "spanId": "0123456789abcdef"}, {"observedTimeUnixNano": "5", "severityNumber": "-9", "attributes": {}, "traceId": "0123456789abcdef0123456789abcdeg", "flags": 0}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"scope": "123", "logRecords": ["s", {"timeUnixNano": "", "observedTimeUnixNano": "5", "attributes": 1, "eventName": -0.0, "body": 1}, {"observedTimeUnixNano": "x", "severityNumber": false, "droppedAttributesCount": 1.5, "flags": "1.25", "eventName": "1.25"}]}, {"scope": "aGVsbG8=", "logRecords": [{"timeUnixNano": 1.5, "severityText": -0.0, "droppedAttributesCount": "a", "traceId": "", "flags": "aGVsbG8=", "eventName": "s"}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": 9223372036854775808, "logRecords": [{"observedTimeUnixNano": 5, "severityNumber": "aGVsbG8=", "attributes": "aGVsbG8=", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": null, "flags": "aGVsbG8="}]}]}, {"schemaUrl": 9223372036854775808, "scopeLogs": [{"scope": {"name": "", "version": -0.0, "attributes": [], "droppedAttributesCount": "1.25"}, "schemaUrl": null, "logRecords": {}}]}]} +{"resourceLogs": [{"schemaUrl": null, "scopeLogs": [{"scope": {"name": "s", "version": false, "attributes": [], "droppedAttributesCount": "abc"}, "logRecords": [{"timeUnixNano": "", "severityNumber": "abc", "severityText": -0.0, "traceId": "", "spanId": "", "flags": "", "eventName": "abc"}, {"timeUnixNano": 1640995200000000000, "severityText": true, "attributes": {}, "droppedAttributesCount": "s", "traceId": "0123456789abcdef0123456789abcdeg"}, {"timeUnixNano": "9x", "observedTimeUnixNano": "", "severityNumber": false, "severityText": "a", "attributes": {}, "traceId": "", "eventName": 1}]}]}, {"scopeLogs": [{"logRecords": []}, {"logRecords": [{"severityNumber": "abc", "severityText": 9223372036854775808, "droppedAttributesCount": 9223372036854775808}]}, {}]}]} +{"resourceLogs": [{"scopeLogs": []}, {"schemaUrl": -0.0, "scopeLogs": [{"scope": -0.0, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "severityNumber": -5, "droppedAttributesCount": true}]}, {"logRecords": 0}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": 9223372036854775808, "scopeLogs": [{"scope": "123"}, {"scope": "a", "logRecords": []}, {"scope": null, "logRecords": [{"severityText": "abc", "droppedAttributesCount": -0.0, "spanId": "0123456789abcdef", "body": "s"}]}]}]} +{"resourceLogs": [{"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": "abc", "scopeLogs": [{"scope": {"name": 1.5, "version": 9223372036854775808, "attributes": [{"key": null, "value": {"arrayValue": 9223372036854775808}}], "droppedAttributesCount": 9223372036854775808}, "logRecords": [{"severityText": "a", "attributes": {}, "droppedAttributesCount": "-9", "body": ""}, {"observedTimeUnixNano": "x", "severityNumber": 1, "severityText": 1.5, "attributes": [{"key": {"stringValue": "w"}}, {"key": "k2", "value": {"a": "aGVsbG8=", "c": ""}}, {"key": "k", "value": {"boolValue": 0}}], "droppedAttributesCount": "123", "flags": "1.25", "eventName": "1.25", "body": null}, {"droppedAttributesCount": "abc", "flags": "-9", "body": -5}]}, {"logRecords": "123"}]}, {"resource": -5, "scopeLogs": "aGVsbG8="}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": 1, "logRecords": []}, {"scope": {"name": -0.0, "version": false, "attributes": [], "droppedAttributesCount": "aGVsbG8="}, "logRecords": []}, {"logRecords": [{"observedTimeUnixNano": "5", "severityNumber": null, "severityText": true, "attributes": [{"key": "k", "value": {"boolValue": 1}}, {"key": {"stringValue": "w"}, "value": {"boolValue": 1.5}}, {"key": "", "value": {}}], "droppedAttributesCount": "aGVsbG8=", "eventName": "abc"}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "5", "severityText": -5, "eventName": "aGVsbG8="}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "5", "severityText": "", "droppedAttributesCount": -5, "traceId": "", "spanId": null, "flags": "", "body": [{"": -5}, {"x": ""}]}]}]}]} +{"resourceLogs": [{"resource": -0.0, "scopeLogs": [{"logRecords": {}}, {"scope": {}, "logRecords": [{"observedTimeUnixNano": 5, "severityNumber": 1, "severityText": 1, "attributes": {}, "flags": null, "body": null}, {"observedTimeUnixNano": "", "severityText": true, "flags": false, "eventName": "s"}]}]}, {"resource": 1.5, "schemaUrl": 1.5, "scopeLogs": [{"scope": {"name": "", "version": "123", "attributes": {}, "droppedAttributesCount": -0.0}, "logRecords": [{"timeUnixNano": 1.5, "severityText": "aGVsbG8=", "attributes": -5, "eventName": "s", "body": {}}, {"severityText": 9223372036854775808, "attributes": 1, "droppedAttributesCount": "1.25", "spanId": "0123", "flags": "abc"}]}, {"logRecords": []}, {"scope": {"name": 9223372036854775808, "version": "abc", "attributes": [], "droppedAttributesCount": "s"}, "logRecords": []}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"value": {"doubleValue": 1.5}}, {"key": "", "value": ["aGVsbG8=", {"stringValue": "-9"}]}, {"key": "k2", "value": {"doubleValue": -5}}], "droppedAttributesCount": -0.0}, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": 0, "scopeLogs": [{"schemaUrl": -5, "logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "x", "severityText": "-9", "attributes": "s", "droppedAttributesCount": true, "flags": -5, "eventName": -5}, {"observedTimeUnixNano": "x", "attributes": "s", "droppedAttributesCount": "123", "flags": true, "body": {"stringValue": "a"}}]}, {"scope": "", "logRecords": [{"severityText": true, "droppedAttributesCount": false}, {"observedTimeUnixNano": "", "attributes": "aGVsbG8=", "droppedAttributesCount": 9223372036854775808, "spanId": "", "flags": "123"}, {"observedTimeUnixNano": "", "attributes": 0, "droppedAttributesCount": "abc", "traceId": 5, "spanId": "", "flags": -5, "body": -5}]}]}]} +{"resourceLogs": [{"schemaUrl": 9223372036854775808, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"scope": {}, "logRecords": []}, {"scope": {}, "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"severityText": "", "attributes": {}, "droppedAttributesCount": "123"}, {"severityNumber": 0, "severityText": 1.5, "attributes": true, "droppedAttributesCount": 9223372036854775808, "traceId": 5, "flags": 1, "body": "1.25"}, {"timeUnixNano": "99999999999999999999999", "traceId": "", "flags": false, "body": [{"bytesValue": "1.25"}, "abc"]}]}, {"scope": {"name": 0, "version": "s", "attributes": ["abc", {"key": {"intValue": 1}}], "droppedAttributesCount": 1}, "schemaUrl": -5, "logRecords": [{"timeUnixNano": "", "severityNumber": 0, "severityText": "a", "droppedAttributesCount": 9223372036854775808, "spanId": null, "flags": "", "eventName": "-9"}]}]}, {"scopeLogs": [{"schemaUrl": -5, "logRecords": [{"severityNumber": "123", "droppedAttributesCount": -5, "traceId": "0123456789abcdef0123456789abcdef"}, {"timeUnixNano": 1.5, "observedTimeUnixNano": 5, "severityText": false, "attributes": [{}, {"key": null}, {"key": "k2", "value": {"doubleValue": "1.25"}}], "droppedAttributesCount": "", "traceId": "", "spanId": "", "eventName": null}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "aGVsbG8="}, "scopeLogs": [{}, {"schemaUrl": 0, "logRecords": [{"severityNumber": "aGVsbG8=", "traceId": 5, "flags": null}, {"observedTimeUnixNano": 5, "attributes": true, "droppedAttributesCount": null, "traceId": "", "flags": -5, "body": {"doubleValue": "a"}}, {"observedTimeUnixNano": "5", "severityNumber": null, "severityText": false, "attributes": {}, "droppedAttributesCount": "", "spanId": ""}]}, {"scope": 0, "schemaUrl": "abc", "logRecords": [{"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "x", "severityNumber": false, "attributes": {}, "traceId": "0123456789abcdef0123456789abcdef", "eventName": "1.25", "body": {}}, {"attributes": {}, "droppedAttributesCount": "1.25", "traceId": 5, "flags": 0, "eventName": ""}, {"timeUnixNano": -1, "attributes": [{"key": null, "value": {"doubleValue": "a"}}, {"key": "k2", "value": "aGVsbG8="}, {"key": 5, "value": "plain"}], "spanId": "", "flags": "s", "eventName": "-9"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": "123", "scopeLogs": [{"logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "spanId": "0123456789abcdef", "eventName": 9223372036854775808, "body": {"doubleValue": {"stringValue": "123"}}}, {"observedTimeUnixNano": "x", "severityText": "a", "attributes": [{"value": {"bytesValue": "s"}}, {}, {"key": "", "value": {"": null}}], "droppedAttributesCount": 0, "spanId": "0123456789abcdef", "flags": false, "body": "aGVsbG8="}, {"timeUnixNano": 1640995200000000000, "severityNumber": 1.5, "attributes": {}, "spanId": "0123", "flags": -0.0, "eventName": "aGVsbG8=", "body": "s"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": "1.25", "scopeLogs": "aGVsbG8="}]} +{"resourceLogs": [{"scopeLogs": [{"scope": -5, "schemaUrl": null, "logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": "5", "flags": 9223372036854775808, "eventName": "abc"}]}]}, {"scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "5", "severityNumber": "1.25", "attributes": {}, "traceId": 5, "spanId": "", "flags": 1.5, "body": 1.5}, {"observedTimeUnixNano": "", "flags": null, "eventName": "1.25"}, {"severityNumber": "s", "attributes": 9223372036854775808, "droppedAttributesCount": 9223372036854775808, "traceId": "0123456789abcdef0123456789abcdef", "flags": "", "eventName": "aGVsbG8="}]}]}, {"scopeLogs": [{"logRecords": [{"attributes": {}, "flags": 0, "body": "s"}, {"observedTimeUnixNano": "5", "severityNumber": "123"}]}, {"schemaUrl": true, "logRecords": [{"observedTimeUnixNano": 5, "severityNumber": -5, "traceId": "0123456789abcdef0123456789abcdeg", "flags": -5, "eventName": "123"}]}]}]} +{"resourceLogs": [{"resource": null, "scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": 5, "droppedAttributesCount": 0, "flags": "s", "eventName": "1.25", "body": -5}, {"timeUnixNano": "1640995200000000000", "attributes": {}, "traceId": "0123456789abcdef0123456789abcdef", "flags": "aGVsbG8=", "body": {"intValue": "abc"}}, {"attributes": {}, "droppedAttributesCount": "123", "body": {}}]}]}, {"scopeLogs": [{"scope": {"name": "abc", "version": "", "attributes": {}, "droppedAttributesCount": 0}, "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": null, "scopeLogs": [{"logRecords": {}}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"timeUnixNano": "1640995200000000000", "severityNumber": "aGVsbG8=", "severityText": 1.5, "droppedAttributesCount": 1, "traceId": 5}, {"timeUnixNano": -1, "observedTimeUnixNano": "5", "droppedAttributesCount": 1.5, "flags": "abc"}, {"observedTimeUnixNano": "x", "severityNumber": "123", "attributes": [], "body": {"doubleValue": "a"}}]}]}, {"resource": {"attributes": {}, "droppedAttributesCount": null}, "scopeLogs": [{"logRecords": {}}, {"scope": {}, "logRecords": []}, {}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": null, "schemaUrl": "aGVsbG8=", "logRecords": [{"observedTimeUnixNano": "", "severityText": "s", "attributes": [{"key": 5, "value": {"boolValue": ""}}], "droppedAttributesCount": 0, "traceId": 5, "spanId": "0123456789abcdef", "flags": 0, "body": 1}, {"attributes": "1.25", "traceId": 5}, {"attributes": [{"key": "k2", "value": {"doubleValue": "aGVsbG8="}}, {"key": null, "value": {"kvlistValue": {"a": {"arrayValue": {"values": [["aGVsbG8=", null], 9223372036854775808]}}, "b": 1}}}], "droppedAttributesCount": -0.0, "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef"}]}]}, {"resource": 1.5, "schemaUrl": "aGVsbG8=", "scopeLogs": [{"scope": {}, "schemaUrl": 1, "logRecords": []}, {"logRecords": []}, {"scope": true, "schemaUrl": "a", "logRecords": [{"observedTimeUnixNano": 5, "severityText": 0, "attributes": [{"key": {"stringValue": "w"}, "value": "plain"}], "droppedAttributesCount": -5, "traceId": "zz", "spanId": "", "flags": "s"}, {"timeUnixNano": 1.5, "observedTimeUnixNano": "", "attributes": [{"key": "k", "value": "plain"}], "traceId": "0123456789abcdef0123456789abcdeg", "flags": "1.25", "body": {"b": {"boolValue": 0}}}, {"observedTimeUnixNano": "", "spanId": null, "flags": "aGVsbG8=", "eventName": "s", "body": null}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": "-9", "scopeLogs": [{"scope": {}, "logRecords": [{"severityNumber": false, "traceId": "", "flags": "s"}]}]}, {"resource": "aGVsbG8=", "schemaUrl": "", "scopeLogs": [{"logRecords": [{"severityNumber": -0.0, "attributes": {}, "droppedAttributesCount": null, "spanId": "0123"}, {"observedTimeUnixNano": "5", "droppedAttributesCount": 1.5, "traceId": 5, "spanId": "", "body": true}]}, {"schemaUrl": null, "logRecords": [{"timeUnixNano": 1.5, "observedTimeUnixNano": "5", "severityNumber": false, "attributes": 9223372036854775808}, {"severityNumber": "aGVsbG8=", "attributes": {}, "spanId": null, "flags": false}]}]}]} +{"resourceLogs": [{}]} +{"resourceLogs": [{"scopeLogs": [{}]}, {"schemaUrl": "abc", "scopeLogs": [{"logRecords": [{"severityNumber": "123", "severityText": "123", "attributes": {}, "droppedAttributesCount": -0.0, "eventName": -0.0, "body": {"arrayValue": []}}]}, {"scope": {}, "logRecords": []}, {"logRecords": [{"observedTimeUnixNano": "x", "severityText": -5, "traceId": "0123456789abcdef0123456789abcdef", "spanId": ""}, {"severityText": "aGVsbG8=", "attributes": [], "spanId": null, "flags": "aGVsbG8="}, {"timeUnixNano": "9x", "observedTimeUnixNano": 5, "traceId": "", "eventName": null}]}]}]} +{"resourceLogs": [{"resource": -5, "schemaUrl": 0, "scopeLogs": []}]} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": "123"}, "scopeLogs": [{"logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": "", "severityNumber": "aGVsbG8=", "severityText": "aGVsbG8=", "droppedAttributesCount": "123", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef", "flags": false, "eventName": "123", "body": {"a": {"int": ""}}}, {"severityNumber": "abc", "severityText": -5, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123456789abcdef", "flags": "1.25", "eventName": "abc"}, {"observedTimeUnixNano": 5, "severityNumber": "-9", "severityText": false, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123"}]}, {"schemaUrl": 9223372036854775808, "logRecords": {}}]}]} +{"resourceLogs": [{"resource": 1.5, "scopeLogs": [1, {"scope": -0.0}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": "aGVsbG8=", "logRecords": [{"traceId": 5, "spanId": null, "flags": "abc", "eventName": "", "body": {}}]}, {}, {"scope": {}, "logRecords": []}]}, {"resource": 1, "scopeLogs": [{"logRecords": [{"timeUnixNano": 1640995200000000000, "traceId": "0123456789abcdef0123456789abcdeg", "eventName": "123"}, {"observedTimeUnixNano": "", "droppedAttributesCount": 1, "eventName": -0.0}]}, {"scope": 0, "logRecords": []}]}]} +{"resourceLogs": [{"resource": -0.0, "schemaUrl": 9223372036854775808, "scopeLogs": [{"scope": "-9", "logRecords": [{"severityNumber": "123", "flags": "123", "eventName": true, "body": "1.25"}]}]}, {"schemaUrl": "", "scopeLogs": [{"scope": "1.25", "logRecords": []}, {"scope": {"name": -5, "version": -5, "attributes": [{"key": "k", "value": {}}], "droppedAttributesCount": ""}, "schemaUrl": 1, "logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": 5, "severityText": true, "attributes": 9223372036854775808, "traceId": "", "eventName": null, "body": {"stringValue": "123"}}, {"droppedAttributesCount": "aGVsbG8=", "traceId": "zz", "flags": "", "body": "-9"}]}, {"logRecords": [""]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": "s", "scopeLogs": [{"schemaUrl": false, "logRecords": []}, {"logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": 5, "droppedAttributesCount": null, "traceId": "", "spanId": "0123456789abcdef", "eventName": ""}]}, {"schemaUrl": 9223372036854775808, "logRecords": "aGVsbG8="}]}]} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": "-9"}, "scopeLogs": null}, {"resource": {"attributes": [{"key": 5, "value": {"boolValue": "a"}}, {"key": null, "value": {"boolValue": 1.5}}, {"key": "k2"}], "droppedAttributesCount": null}, "scopeLogs": []}]} +{"resourceLogs": [{"schemaUrl": 9223372036854775808, "scopeLogs": [{"scope": {"name": false, "version": null, "attributes": [{"key": {"stringValue": "w"}, "value": {"boolValue": -0.0}}, {"key": {"intValue": 1}, "value": {"doubleValue": 0}}, {"key": {"intValue": 1}, "value": {"doubleValue": "abc"}}], "droppedAttributesCount": ""}}, {"schemaUrl": "a", "logRecords": [{"severityNumber": "aGVsbG8=", "severityText": "1.25", "spanId": null, "eventName": "-9"}]}]}]} +{"resourceLogs": [{"resource": "1.25", "schemaUrl": "123", "scopeLogs": [{"logRecords": [{"severityText": 1.5, "traceId": "0123456789abcdef0123456789abcdeg", "flags": "aGVsbG8=", "eventName": "123"}, {"observedTimeUnixNano": "x", "severityNumber": "123", "attributes": [{"key": "", "value": ""}, {"key": "k2", "value": {"kvlistValue": true}}], "droppedAttributesCount": "abc", "traceId": "0123456789abcdef0123456789abcdeg", "flags": 1, "eventName": "aGVsbG8=", "body": {}}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": "aGVsbG8="}, "scopeLogs": []}, {"scopeLogs": "s"}]} +{"resourceLogs": [{"schemaUrl": true, "scopeLogs": [{"logRecords": ""}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": 5, "value": {"intValue": "abc"}}, {"key": "", "value": [-5]}], "droppedAttributesCount": "s"}, "scopeLogs": [{"scope": {}, "schemaUrl": false, "logRecords": [{"traceId": "", "spanId": null}, {"severityNumber": true, "attributes": {}, "traceId": "0123456789abcdef0123456789abcdeg", "flags": 9223372036854775808}]}]}]} +{"resourceLogs": [{"resource": 1, "scopeLogs": [{"logRecords": []}, {"schemaUrl": -5, "logRecords": [{"observedTimeUnixNano": "", "attributes": {}, "droppedAttributesCount": "1.25", "traceId": 5, "body": {}}, {"severityText": "s", "attributes": {}, "droppedAttributesCount": null, "traceId": "", "spanId": "0123", "flags": "123", "body": {}}, {"observedTimeUnixNano": "", "traceId": 5}]}, {"scope": {}, "logRecords": [{"flags": "a", "eventName": "aGVsbG8="}, {"timeUnixNano": 1.5, "attributes": 0, "droppedAttributesCount": false, "body": 9223372036854775808}, {"timeUnixNano": 1.5, "droppedAttributesCount": "s", "flags": null, "eventName": -5}]}]}, {"scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": [{}, 0]}]} +{"resourceLogs": [{"resource": "s", "scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": []}, {"resource": {"attributes": {}, "droppedAttributesCount": "-9"}, "scopeLogs": [{"logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "spanId": "", "flags": false, "eventName": -0.0, "body": {"str": {"str": ""}}}, {"timeUnixNano": "9x", "severityNumber": null, "traceId": "0123456789abcdef0123456789abcdeg", "spanId": ""}]}]}]} +{"resourceLogs": [{"schemaUrl": "", "scopeLogs": [{"schemaUrl": "123", "logRecords": []}, {"scope": {}, "logRecords": [{"timeUnixNano": "9x", "severityNumber": -5, "attributes": "aGVsbG8=", "traceId": ""}]}, {"scope": 1, "logRecords": []}]}]} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": 0}, "scopeLogs": ["123", {"scope": false, "logRecords": [{"timeUnixNano": 1640995200000000000, "severityText": -0.0, "spanId": null, "body": 1.5}, {"observedTimeUnixNano": 5, "severityNumber": -0.0, "attributes": {}, "traceId": 5, "flags": 0, "eventName": 1, "body": {}}, {"severityText": "s", "traceId": "zz", "spanId": "0123"}]}, {"logRecords": {}}]}]} +{"resourceLogs": [{"scopeLogs": [{"scope": -0.0, "logRecords": [{"timeUnixNano": "99999999999999999999999", "severityNumber": "1.25", "severityText": 1.5, "attributes": "123", "spanId": "", "eventName": "123", "body": ""}]}]}]} +{"resourceLogs": [{"resource": "s", "schemaUrl": "123", "scopeLogs": [{"scope": 1.5, "logRecords": true}, {"scope": {}, "logRecords": [{"observedTimeUnixNano": 5, "spanId": null, "eventName": "123", "body": []}, {"observedTimeUnixNano": "", "attributes": [], "traceId": "zz", "flags": true}, {"observedTimeUnixNano": "5", "severityText": "abc", "attributes": 1, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "", "eventName": 9223372036854775808, "body": false}]}, {"scope": {"name": true, "version": "-9", "attributes": {}, "droppedAttributesCount": "-9"}}]}, {"scopeLogs": [{"schemaUrl": "1.25"}, {"scope": {"name": 1.5, "version": "", "attributes": {}, "droppedAttributesCount": "-9"}, "logRecords": [{"droppedAttributesCount": 1, "spanId": null, "eventName": "abc"}]}, {"scope": {}, "logRecords": [{"severityNumber": "123", "spanId": ""}, {"timeUnixNano": "9x", "severityNumber": "a", "attributes": 1.5, "droppedAttributesCount": 9223372036854775808, "spanId": null, "flags": 1.5, "eventName": null, "body": "aGVsbG8="}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": 5, "severityText": "-9", "droppedAttributesCount": "1.25", "flags": "1.25", "body": {"int": 1.5}}]}]}]} +{"resourceLogs": [{"schemaUrl": -5, "scopeLogs": [{"scope": "123", "logRecords": []}, {"logRecords": [{"attributes": "123", "droppedAttributesCount": "1.25", "spanId": "0123456789abcdef", "eventName": 9223372036854775808}, {"timeUnixNano": 1640995200000000000, "severityText": "1.25", "attributes": "123", "droppedAttributesCount": "-9", "spanId": "0123", "flags": "aGVsbG8=", "eventName": "", "body": {"stringValue": 0}}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": 5, "severityText": 1.5, "traceId": 5, "spanId": "", "flags": "abc"}]}, {"schemaUrl": 9223372036854775808}]}, {"resource": {"attributes": {}, "droppedAttributesCount": 9223372036854775808}, "scopeLogs": [{"schemaUrl": "aGVsbG8=", "logRecords": {}}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": null}, "scopeLogs": [{"scope": null, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "severityNumber": true, "severityText": "1.25", "attributes": 1.5, "spanId": "0123", "eventName": -0.0, "body": {"doubleValue": "abc"}}, {"timeUnixNano": 1640995200000000000, "attributes": [], "droppedAttributesCount": "a", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef", "flags": -0.0, "eventName": "1.25", "body": true}]}, {"logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "eventName": "aGVsbG8="}]}]}]} +{"resourceLogs": [{"resource": "a", "scopeLogs": [{"logRecords": []}, {"scope": {"name": 1.5, "version": false, "attributes": [{"key": null}, {"key": "k2", "value": {"int": {"stringValue": 1.5}}}, {"key": "k2", "value": "plain"}], "droppedAttributesCount": null}, "logRecords": [{"timeUnixNano": 1.5, "observedTimeUnixNano": "", "severityNumber": "1.25", "eventName": -0.0}, {"timeUnixNano": "9x", "severityNumber": 9223372036854775808, "severityText": "", "eventName": "a"}]}, {"scope": {}, "logRecords": [{"severityNumber": -5, "attributes": true, "droppedAttributesCount": "", "traceId": "0123456789abcdef0123456789abcdef"}, {"attributes": {}, "droppedAttributesCount": "1.25", "traceId": "zz", "spanId": "0123456789abcdef", "body": {}}, {"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "", "severityNumber": 9223372036854775808, "body": {}}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{}]} +{"resourceLogs": [{"resource": "123", "schemaUrl": true, "scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": 1.5, "attributes": [{"key": null, "value": 0}, {"key": "", "value": {"stringValue": "a"}}], "traceId": "", "flags": "abc"}, {"timeUnixNano": -1, "severityNumber": 9223372036854775808, "severityText": "1.25", "droppedAttributesCount": -5, "traceId": "0123456789abcdef0123456789abcdeg", "flags": 1, "eventName": 1.5}, {"observedTimeUnixNano": "5", "severityNumber": -5, "attributes": true, "spanId": "0123", "eventName": 1}]}, {"logRecords": [{"timeUnixNano": "9x", "severityNumber": "1.25", "droppedAttributesCount": "1.25", "traceId": "zz", "spanId": "0123", "body": true}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": -0.0}, "schemaUrl": "aGVsbG8=", "scopeLogs": [{"scope": -5}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": "-9", "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": -0.0, "scopeLogs": [{"scope": "aGVsbG8=", "logRecords": []}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [1, {"observedTimeUnixNano": "5", "eventName": null, "body": {"kvlistValue": 9223372036854775808}}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "", "severityNumber": "", "droppedAttributesCount": 1.5, "traceId": 5, "eventName": "s"}]}]}, {"schemaUrl": "a", "scopeLogs": [{"scope": {}, "schemaUrl": -0.0, "logRecords": []}, {"schemaUrl": true, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "x", "severityNumber": "1.25", "severityText": 1.5, "attributes": [{"key": 5, "value": [{"x": "a"}]}], "droppedAttributesCount": true, "traceId": "zz"}]}]}]} +{"resourceLogs": [{"scopeLogs": true}]} +{"resourceLogs": [{}, {"resource": {"attributes": {}, "droppedAttributesCount": true}, "scopeLogs": []}]} +{"resourceLogs": [{"resource": {"attributes": [], "droppedAttributesCount": ""}, "scopeLogs": [{"logRecords": []}]}, {"resource": -5, "schemaUrl": false, "scopeLogs": [{"logRecords": [{"timeUnixNano": 1.5, "observedTimeUnixNano": "", "severityNumber": 1.5, "spanId": "", "flags": true, "body": ""}, {"timeUnixNano": "", "severityNumber": true, "droppedAttributesCount": 0, "spanId": "0123456789abcdef"}]}, {"scope": {"name": "a", "version": -0.0, "attributes": {}, "droppedAttributesCount": "abc"}, "schemaUrl": 0, "logRecords": []}]}]} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": 0}, "scopeLogs": [{"scope": {}, "logRecords": []}]}, {"scopeLogs": [{"scope": {"name": 9223372036854775808, "version": -0.0, "attributes": [], "droppedAttributesCount": null}, "schemaUrl": "", "logRecords": [{"timeUnixNano": "99999999999999999999999", "severityNumber": "1.25", "traceId": "0123456789abcdef0123456789abcdeg"}, {"observedTimeUnixNano": 5, "severityText": false, "attributes": [{"key": {"intValue": 1}, "value": ""}, {"key": null}, {"key": "k", "value": {"arrayValue": "aGVsbG8="}}], "traceId": "zz", "spanId": null}, {"attributes": "s", "flags": true, "eventName": "a"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": 5}], "droppedAttributesCount": 1}, "schemaUrl": "1.25", "scopeLogs": [{"scope": "aGVsbG8=", "logRecords": [{"timeUnixNano": "", "severityNumber": "", "severityText": 9223372036854775808, "attributes": [{"key": {"intValue": 1}, "value": 1.5}, {"key": null, "value": "plain"}], "eventName": "-9", "body": "aGVsbG8="}]}]}, {"resource": {"attributes": [{"key": 5}], "droppedAttributesCount": "a"}, "scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "x", "severityNumber": "-9", "severityText": -0.0, "droppedAttributesCount": 0, "flags": "123", "eventName": "-9"}, {"timeUnixNano": 1640995200000000000, "severityNumber": "-9", "severityText": 1.5, "attributes": "abc", "droppedAttributesCount": -0.0, "traceId": "0123456789abcdef0123456789abcdef", "spanId": null}, {"severityText": "a", "eventName": "", "body": true}]}, {"schemaUrl": "-9", "logRecords": [{"attributes": {}, "droppedAttributesCount": "abc", "eventName": "", "body": {"x": "abc"}}, {"severityNumber": true, "severityText": 9223372036854775808, "attributes": {}, "droppedAttributesCount": -5, "body": false}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "x", "severityText": false, "droppedAttributesCount": 9223372036854775808, "traceId": "zz", "flags": 9223372036854775808, "eventName": 9223372036854775808, "body": 0}]}, {"logRecords": [{"observedTimeUnixNano": "x", "severityNumber": 1.5, "severityText": "1.25", "droppedAttributesCount": "1.25", "traceId": "zz", "spanId": null, "eventName": "-9", "body": null}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": -0.0, "scopeLogs": [{"scope": false, "schemaUrl": "1.25", "logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": "", "attributes": [], "flags": "abc", "body": "123"}, {"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "x", "severityText": -0.0, "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123", "flags": 9223372036854775808, "eventName": "-9"}]}]}, {"scopeLogs": [{"scope": {}, "schemaUrl": true, "logRecords": [{"observedTimeUnixNano": "5", "severityNumber": "s", "severityText": 1, "traceId": "0123456789abcdef0123456789abcdef", "flags": "1.25"}, {"severityNumber": 1}]}, {"schemaUrl": 1.5, "logRecords": [{"timeUnixNano": 1.5, "observedTimeUnixNano": 5, "attributes": [{"key": 5, "value": {"arrayValue": "abc"}}, {"value": {"bytesValue": 1.5}}], "flags": "1.25", "eventName": "a", "body": {}}, {"severityNumber": 1, "severityText": "123", "traceId": "", "flags": "-9", "body": ["aGVsbG8=", [{"": "123"}], {"doubleValue": "a"}]}]}]}]} +{"resourceLogs": [{"resource": {"attributes": [{"key": 5}], "droppedAttributesCount": "1.25"}, "scopeLogs": []}, {"resource": 1.5, "scopeLogs": [{"scope": {}, "schemaUrl": 1, "logRecords": []}, {"scope": "123", "logRecords": []}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": -0.0}, "scopeLogs": [{"scope": 0, "logRecords": -5}]}, {"resource": {"attributes": {}, "droppedAttributesCount": 1.5}, "schemaUrl": "s", "scopeLogs": [{"logRecords": [null]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": "", "scopeLogs": []}, {"resource": "1.25"}]} +{"resourceLogs": [{"scopeLogs": [{"scope": -5, "schemaUrl": false, "logRecords": [{"attributes": 0, "droppedAttributesCount": 0, "traceId": "zz", "body": 9223372036854775808}]}]}, {"schemaUrl": 1.5, "scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "1.25"}, "schemaUrl": "1.25", "scopeLogs": []}, {"resource": true, "scopeLogs": [{"scope": 1.5, "logRecords": []}, {"logRecords": [{"observedTimeUnixNano": 5, "severityText": "a", "traceId": 5, "spanId": "0123", "flags": "", "eventName": 1.5, "body": {"doubleValue": 0}}, {"timeUnixNano": "1640995200000000000", "attributes": "s", "traceId": 5, "eventName": "abc"}, {"timeUnixNano": "1640995200000000000", "severityNumber": -5, "spanId": "", "eventName": 1.5, "body": {"boolValue": "abc"}}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"schemaUrl": "aGVsbG8=", "logRecords": [{"observedTimeUnixNano": 5, "droppedAttributesCount": 0, "eventName": true, "body": 9223372036854775808}]}, {"scope": {"name": -5, "version": true, "attributes": [{"key": null, "value": {"boolValue": "abc"}}, {"key": ""}, {"key": "", "value": {"bytesValue": "-9"}}], "droppedAttributesCount": 1}, "schemaUrl": "abc", "logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "", "traceId": "", "flags": 9223372036854775808}, {"timeUnixNano": "99999999999999999999999", "severityText": -5, "spanId": "0123"}, {"timeUnixNano": 1.5, "observedTimeUnixNano": "x", "severityText": 1.5, "traceId": "0123456789abcdef0123456789abcdeg", "eventName": null}]}, {"schemaUrl": -0.0, "logRecords": []}]}, {"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": "k2", "value": {"stringValue": "1.25"}}], "droppedAttributesCount": -0.0}, "schemaUrl": "aGVsbG8=", "scopeLogs": [{"scope": {"name": "s", "version": null, "attributes": [{"key": "k2"}], "droppedAttributesCount": "123"}, "logRecords": [{"timeUnixNano": "1640995200000000000", "observedTimeUnixNano": "x", "severityText": "1.25", "droppedAttributesCount": null, "traceId": "0123456789abcdef0123456789abcdef", "spanId": null, "eventName": null}]}]}]} +{"resourceLogs": [{"schemaUrl": 1, "scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "5", "severityNumber": null, "severityText": "123", "droppedAttributesCount": true, "traceId": "zz", "spanId": null, "eventName": "abc"}, {"timeUnixNano": "9x", "attributes": "a", "droppedAttributesCount": "", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef"}]}]}, {"resource": 1, "scopeLogs": [{"logRecords": []}, {"schemaUrl": "abc", "logRecords": []}, {"logRecords": null}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"scopeLogs": [{"logRecords": 1.5}]}, {"scopeLogs": "aGVsbG8="}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": true, "scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": [{"scope": -5, "schemaUrl": "aGVsbG8=", "logRecords": [{"timeUnixNano": "99999999999999999999999", "severityNumber": "a", "severityText": null, "traceId": "0123456789abcdef0123456789abcdef", "body": 9223372036854775808}, {"spanId": "0123456789abcdef", "flags": -5, "body": 9223372036854775808}, {"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "severityText": null, "droppedAttributesCount": "aGVsbG8=", "spanId": null, "eventName": "1.25"}]}]}, {"scopeLogs": [{"logRecords": [-5]}, {"scope": "", "logRecords": [{"observedTimeUnixNano": "5", "severityNumber": true, "traceId": "zz"}]}, {"logRecords": false}]}]} +{"resourceLogs": [{"resource": true, "scopeLogs": [{}, {"scope": {}, "schemaUrl": "1.25", "logRecords": []}, {"scope": "a", "logRecords": []}]}, {"resource": true, "scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": "1640995200000000000", "severityText": "s", "attributes": 0, "traceId": "0123456789abcdef0123456789abcdef", "spanId": null, "flags": "a", "eventName": "abc", "body": {"int": 9223372036854775808}}, {"observedTimeUnixNano": "", "severityNumber": "aGVsbG8=", "severityText": "a", "droppedAttributesCount": "123", "traceId": "zz", "spanId": "", "flags": true}, {"timeUnixNano": -1, "droppedAttributesCount": "1.25", "traceId": 5, "spanId": "0123", "eventName": ""}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": []}, {"schemaUrl": null, "logRecords": []}, {"logRecords": [{"severityText": null, "droppedAttributesCount": 1.5, "traceId": 5, "spanId": "", "body": "abc"}]}]}, {"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"logRecords": [{"severityNumber": "", "droppedAttributesCount": "1.25", "eventName": -5, "body": ""}, {"timeUnixNano": "", "severityNumber": "aGVsbG8=", "severityText": 1.5, "traceId": "zz", "flags": 1.5, "eventName": 0}]}, {"scope": "a", "logRecords": []}]}]} +{"resourceLogs": [{"schemaUrl": 9223372036854775808, "scopeLogs": [{"scope": {"name": "", "version": "abc", "attributes": [], "droppedAttributesCount": "-9"}, "logRecords": []}]}, {"resource": true, "schemaUrl": 0, "scopeLogs": [{"scope": {}}, {"logRecords": {}}, {"scope": "", "schemaUrl": "abc", "logRecords": [{"timeUnixNano": "", "observedTimeUnixNano": "", "flags": 9223372036854775808, "body": true}]}]}]} +{"resourceLogs": []} +{"resourceLogs": []} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"logRecords": []}, -0.0]}]} +{"resourceLogs": [{"schemaUrl": -5, "scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": []}, {"resource": 1, "scopeLogs": []}]} +{"resourceLogs": [{"scopeLogs": [{}, {"logRecords": []}]}, {"scopeLogs": [{"scope": {}, "logRecords": [{"timeUnixNano": "9x", "observedTimeUnixNano": "5", "severityNumber": "123", "severityText": 1.5, "droppedAttributesCount": "-9", "traceId": "", "body": {}}]}]}]} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"scope": {"name": true, "version": "1.25", "attributes": {}, "droppedAttributesCount": "1.25"}, "logRecords": [{"severityText": "a", "droppedAttributesCount": "123", "traceId": "zz"}, {"timeUnixNano": 1640995200000000000, "observedTimeUnixNano": "x", "droppedAttributesCount": null, "flags": 1.5, "body": {}}, {"timeUnixNano": 1640995200000000000, "severityText": "123", "attributes": "s", "droppedAttributesCount": "1.25", "spanId": "", "flags": 0}]}, {"scope": {"name": 0, "version": null, "attributes": [{"key": null, "value": {"boolValue": true}}, {"value": {"arrayValue": 1.5}}, {"key": "", "value": "abc"}], "droppedAttributesCount": "-9"}, "logRecords": []}]}]} +{"resourceLogs": [{"schemaUrl": "1.25", "scopeLogs": [{"logRecords": {}}]}, {"scopeLogs": [{"logRecords": [{"observedTimeUnixNano": "5", "severityText": 9223372036854775808, "attributes": [{"value": {"boolValue": "s"}}], "flags": "-9", "body": {}}, {"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "x", "severityText": 9223372036854775808, "attributes": -0.0, "droppedAttributesCount": null, "spanId": "0123", "body": true}]}]}]} +{"resourceLogs": [{"resource": false, "scopeLogs": [{"logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "severityNumber": -5, "severityText": 1.5, "attributes": [], "traceId": "0123456789abcdef0123456789abcdeg", "flags": 1.5}]}, {"scope": {}, "logRecords": [{"observedTimeUnixNano": "x", "severityText": 1, "attributes": {}, "droppedAttributesCount": true, "flags": "abc", "eventName": "1.25"}]}, {"scope": {}, "schemaUrl": -5, "logRecords": [{"timeUnixNano": 1640995200000000000, "severityText": -5, "droppedAttributesCount": 1.5, "traceId": "0123456789abcdef0123456789abcdef", "flags": "123"}, {"severityNumber": "abc", "severityText": "", "traceId": "0123456789abcdef0123456789abcdeg", "flags": false, "body": 1}, {"timeUnixNano": 1.5, "observedTimeUnixNano": "5", "attributes": {}, "traceId": "", "spanId": "", "eventName": 1.5}]}]}, {"scopeLogs": [{"scope": {}, "logRecords": [{"flags": "a"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": {}, "droppedAttributesCount": "123"}, "scopeLogs": [{"schemaUrl": "1.25", "logRecords": [{"timeUnixNano": 1640995200000000000, "severityText": false, "body": true}, {"observedTimeUnixNano": "5", "severityNumber": "a", "traceId": 5, "eventName": false}]}]}, {"scopeLogs": []}]} +{"resourceLogs": []} +{"resourceLogs": [{"schemaUrl": null, "scopeLogs": []}, {"schemaUrl": "123", "scopeLogs": [{"scope": {}, "schemaUrl": null}, {"scope": {"name": "s", "version": "123", "attributes": [{"key": "k2", "value": {"intValue": 1}}, {"key": "k", "value": {}}], "droppedAttributesCount": "s"}, "logRecords": ["-9", {"severityNumber": "abc", "severityText": "123", "attributes": [{"key": "k2", "value": "plain"}, {"key": {"intValue": 1}, "value": {"boolValue": 1}}], "spanId": null, "flags": "1.25", "eventName": "aGVsbG8="}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": "k2", "value": {"boolValue": -5}}, {"key": "k", "value": {"": 1}}, {"key": {"stringValue": "w"}, "value": {}}], "droppedAttributesCount": "123"}, "scopeLogs": [{"scope": true, "schemaUrl": "-9", "logRecords": [{"severityNumber": "s", "severityText": "1.25", "traceId": "zz", "eventName": "abc", "body": {}}]}, {"scope": "-9", "logRecords": [{"severityNumber": -0.0, "severityText": "1.25", "attributes": "-9", "flags": 1.5, "eventName": "", "body": {"a": false, "b": {"a": 0, "c": [-5, {"b": false}, "-9"]}}}]}]}, {"resource": "1.25", "scopeLogs": [{"schemaUrl": -5, "logRecords": [{"timeUnixNano": "1640995200000000000", "severityNumber": "", "attributes": null, "droppedAttributesCount": false}, {"timeUnixNano": -1, "attributes": {}}, {"severityNumber": "abc", "droppedAttributesCount": "123", "traceId": "0123456789abcdef0123456789abcdeg", "spanId": "0123456789abcdef", "eventName": -0.0}]}, {"logRecords": [{"observedTimeUnixNano": 5, "severityNumber": 0, "flags": "a", "eventName": true}, {"severityNumber": "-9", "attributes": "1.25", "droppedAttributesCount": "s", "eventName": -0.0}]}]}]} +{"resourceLogs": [{"resource": {"attributes": [{"value": {"stringValue": "s"}}, {"key": {"stringValue": "w"}}], "droppedAttributesCount": 1.5}, "scopeLogs": 9223372036854775808}, {"resource": {"attributes": {}, "droppedAttributesCount": "aGVsbG8="}, "scopeLogs": [{"scope": {}, "logRecords": []}, {"logRecords": [{"timeUnixNano": "99999999999999999999999", "attributes": [{"key": "k", "value": {"stringValue": "-9"}}, {"key": null}, {"key": "k", "value": {}}], "droppedAttributesCount": "123", "traceId": "0123456789abcdef0123456789abcdef", "spanId": null, "flags": 0}]}]}]} +{"resourceLogs": [{"scopeLogs": [{"logRecords": [{"severityNumber": null, "attributes": {}, "spanId": "0123456789abcdef", "body": "abc"}, {"timeUnixNano": "1640995200000000000", "severityText": 1.5, "attributes": {}, "body": {"intValue": 1.5}}, {"observedTimeUnixNano": "x", "severityNumber": "1.25", "attributes": {}, "traceId": "0123456789abcdef0123456789abcdeg", "eventName": 9223372036854775808, "body": {"str": "aGVsbG8="}}]}, {"scope": false, "schemaUrl": "abc", "logRecords": ""}, {"schemaUrl": "abc", "logRecords": [{"observedTimeUnixNano": "x", "severityText": null, "attributes": "abc", "flags": "123", "body": {}}]}]}, {"resource": "abc", "scopeLogs": [{"scope": {"name": "1.25", "version": 9223372036854775808, "attributes": {}, "droppedAttributesCount": 1.5}, "logRecords": [{"severityText": 1.5, "attributes": [{"key": null, "value": 9223372036854775808}, {"key": "k", "value": {"bytesValue": "a"}}, {"key": "", "value": [{"doubleValue": {"arrayValue": {}}}, "s", {"intValue": false}]}], "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123456789abcdef", "eventName": 9223372036854775808, "body": "s"}]}, {"scope": "123", "logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": "5", "severityNumber": "s", "spanId": "", "flags": "abc"}]}]}]} +{"resourceLogs": []} +{"resourceLogs": [{"resource": {"attributes": [{"key": null, "value": {"c": {"int": 0}, "b": -0.0}}, {"key": "", "value": {"bytesValue": -5}}], "droppedAttributesCount": "abc"}, "schemaUrl": "s", "scopeLogs": [{"logRecords": [{"timeUnixNano": 1.5, "severityText": true, "attributes": 1.5, "spanId": "0123", "flags": "abc", "body": {"arrayValue": {"values": [{"stringValue": "123"}, {"intValue": 9223372036854775808}]}}}]}, {"logRecords": [{"timeUnixNano": -1, "observedTimeUnixNano": 5, "severityText": "1.25", "traceId": "0123456789abcdef0123456789abcdef", "spanId": "0123456789abcdef", "flags": "a", "eventName": true}, {"severityNumber": "a", "severityText": true, "attributes": [{"key": {"stringValue": "w"}, "value": {"boolValue": "1.25"}}, {"key": "k", "value": {"doubleValue": "abc"}}], "spanId": ""}, {"severityNumber": -5}]}]}, {"resource": {"attributes": {}, "droppedAttributesCount": null}, "scopeLogs": [{"logRecords": [{"timeUnixNano": -1, "severityNumber": true, "severityText": "", "attributes": {}, "droppedAttributesCount": -0.0, "traceId": 5}]}, {"schemaUrl": "1.25", "logRecords": []}, {"logRecords": [{"timeUnixNano": "99999999999999999999999", "observedTimeUnixNano": 5, "severityText": -5, "droppedAttributesCount": 9223372036854775808, "spanId": null, "flags": "a", "body": {}}, {"traceId": "", "flags": "", "eventName": null, "body": 1.5}]}]}]} +{"resourceLogs": [{"resource": {"attributes": [{"key": {"stringValue": "w"}}], "droppedAttributesCount": -5}, "schemaUrl": -0.0, "scopeLogs": [{"schemaUrl": "-9", "logRecords": [{"severityText": true, "attributes": {}, "spanId": "0123", "flags": true, "eventName": false}]}, {"scope": {"name": "s", "version": 1.5, "attributes": [{"key": {"stringValue": "w"}, "value": {"bytesValue": "a"}}], "droppedAttributesCount": "a"}, "schemaUrl": -5}]}, {"scopeLogs": []}]} +{"resourceLogs": []} diff --git a/tests/internal/data/opentelemetry/logs_parity.pb b/tests/internal/data/opentelemetry/logs_parity.pb new file mode 100644 index 00000000000..6a68955b109 Binary files /dev/null and b/tests/internal/data/opentelemetry/logs_parity.pb differ diff --git a/tests/internal/data/opentelemetry/logs_parity_corpus.py b/tests/internal/data/opentelemetry/logs_parity_corpus.py new file mode 100755 index 00000000000..893b0bccc19 --- /dev/null +++ b/tests/internal/data/opentelemetry/logs_parity_corpus.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Generate the corpora compared by the OTLP logs CFL parity tests. + +logs_parity.jsonl holds one OTLP/JSON logs request per line. logs_parity.pb +holds OTLP protobuf logs requests, each one prefixed by its length as a +little-endian 32-bit integer. + +The requests are random and often invalid on purpose: the tests check that +the CFL decoders accept, reject and size every request like the msgpack +decoders do. + +Usage: logs_parity_corpus.py [count] [seed] +""" + +import json +import os +import random +import struct +import sys + +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest + +# Wrapper keys, including prefixes and unknown names +JSON_WRAPPERS = ["stringValue", "boolValue", "intValue", "doubleValue", "bytesValue", + "arrayValue", "kvlistValue", "str", "int", "x", ""] + + +def json_scalar(): + return random.choice([None, True, False, 0, 1, -5, 2**63, 1.5, -0.0, "", "s", "123", + "-9", "1.25", "abc", "aGVsbG8=", "a"]) + + +def json_any_value(depth=0): + choice = random.random() + if depth > 3 or choice < 0.3: + return json_scalar() + if choice < 0.7: + key = random.choice(JSON_WRAPPERS) + if key == "arrayValue": + value = random.choice([ + [json_any_value(depth + 1) for _ in range(random.randint(0, 3))], + {"values": [json_any_value(depth + 1) for _ in range(2)]}, + {}, {"other": 1}, json_scalar()]) + elif key == "kvlistValue": + value = random.choice([ + {"values": json_key_values(depth + 1)}, {}, + {"a": json_any_value(depth + 1), "b": 1}, + json_key_values(depth + 1), json_scalar()]) + elif random.random() < 0.8: + value = json_scalar() + else: + value = json_any_value(depth + 1) + return {key: value} + if choice < 0.85: + return [json_any_value(depth + 1) for _ in range(random.randint(0, 3))] + return {random.choice(["a", "b", "c"]): json_any_value(depth + 1) + for _ in range(random.randint(0, 3))} + + +def json_key_values(depth=0): + entries = [] + for _ in range(random.randint(0, 3)): + entry = {} + if random.random() < 0.9: + entry["key"] = random.choice(["k", "k2", "", None, 5, {"stringValue": "w"}, + {"intValue": 1}]) + if random.random() < 0.85: + if random.random() < 0.2: + entry["value"] = random.choice([{}, json_any_value(depth + 1), "plain"]) + elif random.random() < 0.3: + entry["value"] = json_any_value(depth + 1) + else: + entry["value"] = {random.choice(JSON_WRAPPERS[:7]): json_scalar()} + entries.append(entry if random.random() < 0.95 else json_scalar()) + return entries + + +def json_log_record(): + fields = [ + ("timeUnixNano", lambda: random.choice(["1640995200000000000", "", 1640995200000000000, + -1, 1.5, "9x", "99999999999999999999999"])), + ("observedTimeUnixNano", lambda: random.choice(["5", "x", 5, ""])), + ("severityNumber", json_scalar), + ("severityText", json_scalar), + ("attributes", lambda: random.choice([json_key_values(), {}, json_scalar()])), + ("droppedAttributesCount", json_scalar), + ("traceId", lambda: random.choice(["", "0123456789abcdef0123456789abcdef", "zz", 5, + "0123456789abcdef0123456789abcdeg"])), + ("spanId", lambda: random.choice(["", "0123456789abcdef", "0123", None])), + ("flags", json_scalar), + ("eventName", json_scalar), + ("body", lambda: random.choice([json_any_value(), json_any_value(), {}, json_scalar()])), + ] + record = {} + for name, generate in fields: + if random.random() < 0.45: + record[name] = generate() + return record if random.random() < 0.97 else json_scalar() + + +def json_scope_logs(): + scope_logs = {} + if random.random() < 0.5: + scope_logs["scope"] = random.choice([ + {"name": json_scalar(), "version": json_scalar(), + "attributes": random.choice([json_key_values(), {}, []]), + "droppedAttributesCount": json_scalar()}, + {}, json_scalar()]) + if random.random() < 0.3: + scope_logs["schemaUrl"] = json_scalar() + if random.random() < 0.95: + if random.random() < 0.1: + scope_logs["logRecords"] = random.choice([{}, json_scalar()]) + else: + scope_logs["logRecords"] = [json_log_record() for _ in range(random.randint(0, 3))] + return scope_logs if random.random() < 0.97 else json_scalar() + + +def json_resource_logs(): + resource_logs = {} + if random.random() < 0.5: + resource_logs["resource"] = random.choice([ + {"attributes": random.choice([json_key_values(), {}]), + "droppedAttributesCount": json_scalar()}, + json_scalar()]) + if random.random() < 0.3: + resource_logs["schemaUrl"] = json_scalar() + if random.random() < 0.97: + if random.random() < 0.95: + resource_logs["scopeLogs"] = [json_scope_logs() for _ in range(random.randint(0, 3))] + else: + resource_logs["scopeLogs"] = json_scalar() + return resource_logs if random.random() < 0.97 else json_scalar() + + +def json_request(): + return {"resourceLogs": [json_resource_logs() for _ in range(random.randint(0, 2))]} + + +def pb_any_value(value, depth=0): + choice = random.random() + if choice < 0.08: + return + if depth > 3 or choice < 0.55: + field = random.choice(["string_value", "bool_value", "int_value", "double_value", + "bytes_value"]) + setattr(value, field, { + "string_value": random.choice(["", "s", "x" * 40]), + "bool_value": random.choice([True, False]), + "int_value": random.choice([0, 1, -1, 2**62, -(2**63)]), + "double_value": random.choice([0.0, -1.5, 1e300]), + "bytes_value": random.choice([b"", b"\x00\xff", b"abc"]), + }[field]) + return + if choice < 0.78: + for _ in range(random.randint(0, 3)): + pb_any_value(value.array_value.values.add(), depth + 1) + if not value.array_value.values and random.random() < 0.5: + value.array_value.SetInParent() + return + pb_key_values(value.kvlist_value.values, depth + 1) + if not value.kvlist_value.values: + value.kvlist_value.SetInParent() + + +def pb_key_values(entries, depth=0): + for _ in range(random.randint(0, 3)): + entry = entries.add(key=random.choice(["k", "", "key.name", "k"])) + if random.random() < 0.9: + pb_any_value(entry.value, depth) + + +def pb_log_record(record): + if random.random() < 0.8: + record.time_unix_nano = random.choice([1640995200000000000, 1, 0]) + if random.random() < 0.5: + record.observed_time_unix_nano = random.choice([1640995200000000001, 0]) + if random.random() < 0.5: + record.severity_number = random.choice([0, 9, 24, 25]) + if random.random() < 0.4: + record.severity_text = random.choice(["", "INFO"]) + if random.random() < 0.7: + pb_any_value(record.body) + if random.random() < 0.5: + pb_key_values(record.attributes) + if random.random() < 0.3: + record.dropped_attributes_count = random.randint(0, 3) + if random.random() < 0.3: + record.flags = random.choice([0, 1, 257]) + if random.random() < 0.3: + record.trace_id = random.choice([b"", bytes(16), bytes(range(16)), b"\x01" * 8]) + if random.random() < 0.3: + record.span_id = random.choice([b"", bytes(8), bytes(range(8)), b"\x01" * 4]) + + +def pb_request(): + request = ExportLogsServiceRequest() + for _ in range(random.randint(0, 2)): + resource_logs = request.resource_logs.add() + if random.random() < 0.6: + pb_key_values(resource_logs.resource.attributes) + if random.random() < 0.3: + resource_logs.resource.dropped_attributes_count = random.randint(0, 5) + if random.random() < 0.2: + resource_logs.resource.SetInParent() + if random.random() < 0.3: + resource_logs.schema_url = random.choice(["", "https://r"]) + for _ in range(random.randint(0, 3)): + scope_logs = resource_logs.scope_logs.add() + if random.random() < 0.6: + scope = scope_logs.scope + if random.random() < 0.5: + scope.name = random.choice(["", "n"]) + if random.random() < 0.5: + scope.version = random.choice(["", "1"]) + if random.random() < 0.5: + pb_key_values(scope.attributes) + if random.random() < 0.3: + scope.dropped_attributes_count = random.randint(0, 3) + scope.SetInParent() + if random.random() < 0.3: + scope_logs.schema_url = random.choice(["", "https://s"]) + for _ in range(random.randint(0, 3)): + pb_log_record(scope_logs.log_records.add()) + return request.SerializeToString() + + +def main(): + count = int(sys.argv[1]) if len(sys.argv) > 1 else 200 + random.seed(int(sys.argv[2]) if len(sys.argv) > 2 else 7) + directory = os.path.dirname(os.path.abspath(__file__)) + + with open(os.path.join(directory, "logs_parity.jsonl"), "w") as output: + for _ in range(count): + output.write(json.dumps(json_request()) + "\n") + + with open(os.path.join(directory, "logs_parity.pb"), "wb") as output: + for _ in range(count): + request = pb_request() + output.write(struct.pack(" +#include +#include + +#include +#include +#include +#include + +#include "flb_fuzz_header.h" + +/* + * Feed OTLP/HTTP log payloads to the decoders used by in_opentelemetry, the + * way the input does: the CFL decoders followed by the record encoding, and + * the msgpack decoders. The first byte selects the encoding, the body key and + * the logs_max_size budget, the rest is the request body. + */ + +/* Encode CFL records like the input does once processors are done */ +static void encode_chunk(struct flb_mp_chunk_cobj *chunk) +{ + struct flb_log_event_encoder encoder; + struct flb_mp_chunk_record *record; + + if (chunk == NULL) { + return; + } + + while (flb_mp_chunk_cobj_record_next(chunk, &record) == FLB_MP_CHUNK_RECORD_OK) { + } + + if (flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) == 0) { + chunk->log_encoder = &encoder; + flb_mp_chunk_cobj_encode_append(chunk); + flb_mp_chunk_cobj_count_log_records(chunk); + chunk->log_encoder = NULL; + flb_log_event_encoder_destroy(&encoder); + } + + flb_mp_chunk_cobj_destroy(chunk); +} + +static void fuzz_protobuf(const char *payload, size_t size, + struct flb_opentelemetry_logs_decode_options *options) +{ + size_t record_count; + struct flb_log_event_encoder encoder; + struct flb_opentelemetry_logs_protobuf_cfl *result; + + result = NULL; + if (flb_opentelemetry_logs_protobuf_to_cfl(payload, size, options, &result) == 0) { + flb_opentelemetry_logs_protobuf_cfl_get_encoded_size(result); + encode_chunk(flb_opentelemetry_logs_protobuf_cfl_take_chunk(result)); + } + flb_opentelemetry_logs_protobuf_cfl_destroy(result); + + if (flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) == 0) { + encoder.output_limit = options->output_limit; + flb_opentelemetry_logs_protobuf_to_msgpack(&encoder, payload, size, options, + &record_count); + flb_log_event_encoder_destroy(&encoder); + } +} + +static void fuzz_json(const char *payload, size_t size, + struct flb_opentelemetry_logs_decode_options *options) +{ + int error_status; + struct flb_log_event_encoder encoder; + struct flb_opentelemetry_logs_json_cfl *result; + + result = NULL; + error_status = 0; + if (flb_opentelemetry_logs_json_to_cfl(payload, size, options, &result, + &error_status) == 0) { + flb_opentelemetry_logs_json_cfl_get_encoded_size(result); + encode_chunk(flb_opentelemetry_logs_json_cfl_take_chunk(result)); + } + flb_opentelemetry_logs_json_cfl_destroy(result); + + if (flb_log_event_encoder_init(&encoder, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2) == 0) { + encoder.output_limit = options->output_limit; + error_status = 0; + flb_opentelemetry_logs_json_to_msgpack(&encoder, payload, size, + options->body_key, &error_status); + flb_log_event_encoder_destroy(&encoder); + } +} + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + uint8_t selector; + char *payload; + struct flb_opentelemetry_logs_decode_options options; + + TIMEOUT_GUARD + + /* Set fuzzer-malloc chance of failure */ + flb_malloc_p = 0; + flb_malloc_mod = 25000; + + if (size < 1) { + return 0; + } + selector = data[0]; + MOVE_INPUT(1); + + /* exact-size copy, so reads past the payload are caught */ + payload = malloc(size > 0 ? size : 1); + if (payload == NULL) { + return 0; + } + if (size > 0) { + memcpy(payload, data, size); + } + + options.metadata_key = "otlp"; + options.body_key = (selector & 0x02) ? "log" : NULL; + options.output_limit = (selector >> 2) == 0 ? SIZE_MAX : (size_t) (selector >> 2) * 256; + + if (selector & 0x01) { + fuzz_protobuf(payload, size, &options); + } + else { + fuzz_json(payload, size, &options); + } + + free(payload); + + return 0; +} diff --git a/tests/internal/json.c b/tests/internal/json.c new file mode 100644 index 00000000000..317dec4c9ae --- /dev/null +++ b/tests/internal/json.c @@ -0,0 +1,212 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#include "flb_tests_internal.h" + +#ifdef FLB_HAVE_YYJSON +static struct flb_json_tree *tree_read(const char *json) +{ + return flb_json_tree_read(json, strlen(json)); +} + +static void test_tree_types() +{ + size_t index; + size_t maximum; + struct flb_json_tree *tree; + struct flb_json_node *root; + struct flb_json_node *key; + struct flb_json_node *value; + static const char *names[] = {"n", "b", "u", "s", "r", "t", "a", "o"}; + + tree = tree_read("{\"n\":null,\"b\":true,\"u\":18446744073709551615," + "\"s\":-42,\"r\":1.5,\"t\":\"text\",\"a\":[1,2],\"o\":{}}"); + TEST_CHECK(tree != NULL); + if (tree == NULL) { + return; + } + + root = flb_json_tree_root(tree); + TEST_CHECK(flb_json_node_is_obj(root)); + TEST_CHECK(flb_json_node_obj_size(root) == 8); + + flb_json_node_obj_foreach(root, index, maximum, key, value) { + TEST_CHECK(flb_json_node_is_str(key)); + TEST_CHECK(flb_json_node_get_len(key) == 1); + TEST_CHECK(strncmp(flb_json_node_get_str(key), names[index], 1) == 0); + + switch (index) { + case 0: + TEST_CHECK(flb_json_node_is_null(value)); + break; + case 1: + TEST_CHECK(flb_json_node_is_bool(value) && flb_json_node_get_bool(value)); + break; + case 2: + TEST_CHECK(flb_json_node_is_uint(value) && flb_json_node_is_int(value)); + TEST_CHECK(flb_json_node_get_uint(value) == UINT64_MAX); + break; + case 3: + TEST_CHECK(flb_json_node_is_sint(value) && flb_json_node_is_int(value)); + TEST_CHECK(flb_json_node_get_sint(value) == -42); + break; + case 4: + TEST_CHECK(flb_json_node_is_real(value) && !flb_json_node_is_int(value)); + TEST_CHECK(flb_json_node_get_real(value) == 1.5); + break; + case 5: + TEST_CHECK(flb_json_node_is_str(value)); + TEST_CHECK(flb_json_node_get_len(value) == 4); + TEST_CHECK(strncmp(flb_json_node_get_str(value), "text", 4) == 0); + break; + case 6: + TEST_CHECK(flb_json_node_is_arr(value)); + TEST_CHECK(flb_json_node_arr_size(value) == 2); + break; + case 7: + TEST_CHECK(flb_json_node_is_obj(value)); + TEST_CHECK(flb_json_node_obj_size(value) == 0); + TEST_CHECK(flb_json_node_first(value) == NULL); + break; + } + } + TEST_CHECK(index == 8); + + flb_json_tree_destroy(tree); +} + +static void test_tree_nested_iteration() +{ + size_t total; + size_t index; + size_t maximum; + size_t inner_index; + size_t inner_maximum; + struct flb_json_tree *tree; + struct flb_json_node *root; + struct flb_json_node *entry; + struct flb_json_node *inner; + + /* containers inside arrays must be skipped as a whole */ + tree = tree_read("[[1,[2,3]],{\"k\":[4,{\"x\":5}]},6]"); + TEST_CHECK(tree != NULL); + if (tree == NULL) { + return; + } + + root = flb_json_tree_root(tree); + TEST_CHECK(flb_json_node_arr_size(root) == 3); + + total = 0; + flb_json_node_arr_foreach(root, index, maximum, entry) { + if (index == 0) { + TEST_CHECK(flb_json_node_is_arr(entry)); + flb_json_node_arr_foreach(entry, inner_index, inner_maximum, inner) { + total++; + } + } + else if (index == 1) { + TEST_CHECK(flb_json_node_is_obj(entry)); + inner = flb_json_node_key_value(flb_json_node_first(entry)); + TEST_CHECK(flb_json_node_is_arr(inner)); + TEST_CHECK(flb_json_node_arr_size(inner) == 2); + } + else { + TEST_CHECK(flb_json_node_is_uint(entry)); + TEST_CHECK(flb_json_node_get_uint(entry) == 6); + } + } + TEST_CHECK(index == 3); + TEST_CHECK(total == 2); + + flb_json_tree_destroy(tree); +} + +static void test_tree_invalid_unicode() +{ + struct flb_json_tree *tree; + struct flb_json_node *root; + + /* an unpaired surrogate is replaced with U+FFFD, like flb_pack_json() */ + tree = tree_read("\"a\\uD800b\""); + TEST_CHECK(tree != NULL); + if (tree == NULL) { + return; + } + + root = flb_json_tree_root(tree); + TEST_CHECK(flb_json_node_is_str(root)); + TEST_CHECK(flb_json_node_get_len(root) == 5); + TEST_CHECK(memcmp(flb_json_node_get_str(root), "a\xef\xbf\xbd" "b", 5) == 0); + + flb_json_tree_destroy(tree); +} + +static void test_tree_rejects() +{ + TEST_CHECK(flb_json_tree_read(NULL, 0) == NULL); + TEST_CHECK(tree_read("") == NULL); + TEST_CHECK(tree_read("{\"a\":") == NULL); + TEST_CHECK(tree_read("{} trailing") == NULL); + TEST_CHECK(tree_read("{}{}") == NULL); +} +#endif + +static void test_tree_null_nodes() +{ + TEST_CHECK(flb_json_tree_root(NULL) == NULL); + TEST_CHECK(!flb_json_node_is_null(NULL)); + TEST_CHECK(!flb_json_node_is_str(NULL)); + TEST_CHECK(!flb_json_node_is_obj(NULL)); + TEST_CHECK(!flb_json_node_is_arr(NULL)); + TEST_CHECK(flb_json_node_get_str(NULL) == NULL); + TEST_CHECK(flb_json_node_get_len(NULL) == 0); + TEST_CHECK(flb_json_node_arr_size(NULL) == 0); + TEST_CHECK(flb_json_node_obj_size(NULL) == 0); + TEST_CHECK(flb_json_node_first(NULL) == NULL); + TEST_CHECK(flb_json_node_next(NULL) == NULL); + TEST_CHECK(flb_json_node_key_value(NULL) == NULL); + flb_json_tree_destroy(NULL); +} + +#ifndef FLB_HAVE_YYJSON +static void test_tree_unavailable() +{ + /* without a tree backend callers keep their other way to read JSON */ + TEST_CHECK(flb_json_tree_read("{}", 2) == NULL); +} +#endif + +TEST_LIST = { +#ifdef FLB_HAVE_YYJSON + { "tree_types", test_tree_types }, + { "tree_nested_iteration", test_tree_nested_iteration }, + { "tree_invalid_unicode", test_tree_invalid_unicode }, + { "tree_rejects", test_tree_rejects }, +#else + { "tree_unavailable", test_tree_unavailable }, +#endif + { "tree_null_nodes", test_tree_null_nodes }, + { 0 } +}; diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index ac107fc65e6..a7c886fe1f0 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -27,6 +27,7 @@ // #include "../../plugins/in_opentelemetry/opentelemetry.h" #include +#include #include #include #include @@ -609,6 +610,8 @@ void test_json_payload_get_wrapped_value() #define OTEL_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/logs.json" #define OTEL_TRACES_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/traces.json" #define OTEL_METRICS_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/metrics.json" +#define OTEL_JSON_PARITY_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/logs_parity.jsonl" +#define OTEL_PROTOBUF_PARITY_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/logs_parity.pb" static flb_sds_t test_normalize_json(const char *input) { @@ -1719,6 +1722,515 @@ void test_opentelemetry_cases() flb_free(cases_json); } +/* + * The input encodes OTLP JSON logs through the single-pass CFL decoder. Every + * shared case must produce the same records, or the same error, as the + * generic JSON to msgpack conversion. + */ +static int json_cfl_encode(struct flb_log_event_encoder *encoder, + const char *json, size_t len, size_t limit, int *error_status) +{ + int ret; + struct flb_mp_chunk_cobj *chunk; + struct flb_opentelemetry_logs_json_cfl *result; + struct flb_opentelemetry_logs_decode_options options; + + result = NULL; + options.metadata_key = "otlp"; + options.body_key = NULL; + options.output_limit = limit; + encoder->output_limit = limit; + + ret = flb_opentelemetry_logs_json_to_cfl(json, len, &options, &result, error_status); + if (ret != 0) { + flb_opentelemetry_logs_json_cfl_destroy(result); + return ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED ? ret : -1; + } + + chunk = flb_opentelemetry_logs_json_cfl_take_chunk(result); + if (chunk == NULL) { + flb_opentelemetry_logs_json_cfl_destroy(result); + return -1; + } + + chunk->log_encoder = encoder; + ret = flb_mp_chunk_cobj_encode_append(chunk); + chunk->log_encoder = NULL; + + flb_mp_chunk_cobj_destroy(chunk); + flb_opentelemetry_logs_json_cfl_destroy(result); + + return ret; +} + +/* + * Compare two encoded buffers event by event. Records without a timestamp + * get the current time on each path, any recent time is accepted for them. + */ +static int log_events_equal(char *buffer, size_t size, char *other, size_t other_size) +{ + int ret; + int other_ret; + time_t now; + struct flb_log_event event; + struct flb_log_event other_event; + struct flb_log_event_decoder decoder; + struct flb_log_event_decoder other_decoder; + + now = time(NULL); + flb_log_event_decoder_init(&decoder, buffer, size); + flb_log_event_decoder_init(&other_decoder, other, other_size); + flb_log_event_decoder_read_groups(&decoder, FLB_TRUE); + flb_log_event_decoder_read_groups(&other_decoder, FLB_TRUE); + + while (1) { + ret = flb_log_event_decoder_next(&decoder, &event); + other_ret = flb_log_event_decoder_next(&other_decoder, &other_event); + if (ret != FLB_EVENT_DECODER_SUCCESS || other_ret != FLB_EVENT_DECODER_SUCCESS) { + break; + } + if (flb_time_equal(&event.timestamp, &other_event.timestamp) == FLB_FALSE && + (event.timestamp.tm.tv_sec < now - 3600 || + other_event.timestamp.tm.tv_sec < now - 3600)) { + break; + } + if (msgpack_object_equal(*event.metadata, *other_event.metadata) == false || + msgpack_object_equal(*event.body, *other_event.body) == false) { + break; + } + } + + flb_log_event_decoder_destroy(&decoder); + flb_log_event_decoder_destroy(&other_decoder); + + return ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA && + other_ret == FLB_EVENT_DECODER_ERROR_INSUFFICIENT_DATA; +} + +/* + * Decode one payload through both paths and compare the outcome. A payload + * the CFL decoder reports as unsupported is decoded by the msgpack path in + * the input, so it is equal by construction. + */ +static void json_cfl_check_parity(const char *name, const char *json, size_t limit) +{ + int ret; + int cfl_ret; + int error_status; + int cfl_error_status; + struct flb_log_event_encoder enc; + struct flb_log_event_encoder cfl_enc; + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + flb_log_event_encoder_init(&cfl_enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + enc.output_limit = limit; + + error_status = 0; + cfl_error_status = 0; + ret = flb_opentelemetry_logs_json_to_msgpack(&enc, json, strlen(json), NULL, + &error_status); + cfl_ret = json_cfl_encode(&cfl_enc, json, strlen(json), limit, &cfl_error_status); + + if (cfl_ret != FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED) { + TEST_CHECK_((ret == 0) == (cfl_ret == 0), "%s (limit %zu): result %i vs %i", + name, limit, ret, cfl_ret); + if (ret == 0 && cfl_ret == 0) { + TEST_CHECK_(log_events_equal(enc.output_buffer, enc.output_length, + cfl_enc.output_buffer, cfl_enc.output_length), + "%s (limit %zu): records differ", name, limit); + } + else if (limit == SIZE_MAX) { + TEST_CHECK_(error_status == cfl_error_status, "%s: error %i vs %i", + name, error_status, cfl_error_status); + } + } + + flb_log_event_encoder_destroy(&enc); + flb_log_event_encoder_destroy(&cfl_enc); +} + +/* + * Invalid Unicode is replaced on both paths with the same records. Content + * after the request object is left to the established conversion, which + * accepts it with yyjson. Without yyjson every payload is left to it. + */ +void test_opentelemetry_json_cfl_lenient_payloads() +{ + int ret; + int error_status; + struct flb_log_event_encoder enc; + static const char *invalid_unicode = + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[{\"body\":" + "{\"stringValue\":\"bad \\uD800 surrogate\"}}]}]}]}"; + static const char *trailing = + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[{\"body\":" + "{\"stringValue\":\"text\"}}]}]}]} trailing"; + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + error_status = 0; + ret = json_cfl_encode(&enc, invalid_unicode, strlen(invalid_unicode), SIZE_MAX, + &error_status); +#ifdef FLB_HAVE_YYJSON + TEST_CHECK_(ret == 0 && enc.output_length > 0, + "invalid unicode: CFL result %i (%i)", ret, error_status); +#else + TEST_CHECK_(ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED, + "invalid unicode: CFL result %i", ret); +#endif + flb_log_event_encoder_destroy(&enc); + json_cfl_check_parity("invalid unicode", invalid_unicode, SIZE_MAX); + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + error_status = 0; + ret = json_cfl_encode(&enc, trailing, strlen(trailing), SIZE_MAX, &error_status); + TEST_CHECK_(ret == FLB_OTEL_LOGS_JSON_CFL_UNSUPPORTED, + "trailing content: CFL result %i", ret); + flb_log_event_encoder_destroy(&enc); + +#ifdef FLB_HAVE_YYJSON + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + error_status = 0; + ret = flb_opentelemetry_logs_json_to_msgpack(&enc, trailing, strlen(trailing), + NULL, &error_status); + TEST_CHECK_(ret == 0 && enc.output_length > 0, + "trailing content: msgpack result %i (%i)", ret, error_status); + flb_log_event_encoder_destroy(&enc); +#endif +} + +/* Payloads that exercise the leniency of the established conversion */ +void test_opentelemetry_json_cfl_parity_values() +{ + size_t i; + char json[2048]; + static const char *bodies[] = { + "{\"stringValue\":\"text\"}", + "{\"stringValue\":null}", + "{\"str\":\"prefix of stringValue\"}", + "{\"\":\"empty wrapper key\"}", + "{\"intValue\":\"123\"}", + "{\"intValue\":\"-42abc\"}", + "{\"intValue\":\"not a number\"}", + "{\"intValue\":7}", + "{\"intValue\":-7}", + "{\"intValue\":1.5}", + "{\"doubleValue\":\"1.5\"}", + "{\"doubleValue\":2.25}", + "{\"doubleValue\":3}", + "{\"boolValue\":\"true\"}", + "{\"bytesValue\":\"aGVsbG8=\"}", + "{\"bytesValue\":\"aGVsbG8\"}", + "{\"bytesValue\":\"a\"}", + "{\"arrayValue\":{}}", + "{\"arrayValue\":{\"values\":[{\"intValue\":\"1\"},null,[1,2]]}}", + "{\"arrayValue\":{\"other\":[]}}", + "{\"arrayValue\":[1,{\"stringValue\":\"x\"}]}", + "{\"kvlistValue\":{}}", + "{\"kvlistValue\":{\"values\":[]}}", + "{\"kvlistValue\":{\"a\":1,\"b\":2}}", + "{\"kvlistValue\":{\"values\":[{\"key\":\"k\"}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":\"k\",\"value\":{}}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":\"k\",\"value\":\"plain\"}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":{\"stringValue\":\"w\"}," + "\"value\":{\"intValue\":\"1\"}}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":null,\"value\":{\"boolValue\":true}}]}}", + "{\"kvlistValue\":{\"values\":[{\"key\":5,\"value\":{\"boolValue\":true}}]}}", + "{\"kvlistValue\":{\"values\":[{\"value\":{\"boolValue\":true}}]}}", + "{\"kvlistValue\":{\"values\":[\"not a map\"]}}", + "{\"kvlistValue\":[]}", + "{}", + "{\"plain\":{\"nested\":[1,null,true]},\"other\":\"x\"}", + "{\"custom\":{\"child\":[1,true,null]}}", + "{\"unknownValue\":1}", + "{\"stringValue\":\"a\",\"extra\":1}", + }; + static const char *records[] = { + "{\"timeUnixNano\":\"\"}", + "{\"timeUnixNano\":\"12345678901234567890123456789012345\"}", + "{\"timeUnixNano\":1722904465173450100}", + "{\"timeUnixNano\":-1}", + "{\"timeUnixNano\":1.5}", + "{\"timeUnixNano\":\"12a\"}", + "{\"observedTimeUnixNano\":\"77\"}", + "{\"timeUnixNano\":\"1\",\"observedTimeUnixNano\":5}", + "{\"timeUnixNano\":\"1\",\"observedTimeUnixNano\":\"x\"}", + "{\"timeUnixNano\":\"1\",\"observedTimeUnixNano\":\"18446744073709551615\"}", + "{\"timeUnixNano\":\"1\",\"severityNumber\":{\"a\":[1,null]}}", + "{\"timeUnixNano\":\"1\",\"severityNumber\":null,\"severityText\":5}", + "{\"timeUnixNano\":\"1\",\"flags\":\"257\"}", + "{\"timeUnixNano\":\"1\",\"traceFlags\":3}", + "{\"timeUnixNano\":\"1\",\"flags\":-3}", + "{\"timeUnixNano\":\"1\",\"flags\":\"3x\"}", + "{\"timeUnixNano\":\"1\",\"droppedAttributesCount\":-2}", + "{\"timeUnixNano\":\"1\",\"droppedAttributesCount\":\"2\"}", + "{\"timeUnixNano\":\"1\",\"traceId\":\"\",\"spanId\":\"\"}", + "{\"timeUnixNano\":\"1\",\"traceId\":5}", + "{\"timeUnixNano\":\"1\",\"attributes\":{}}", + "{\"timeUnixNano\":\"1\",\"attributes\":[{\"key\":\"a\",\"value\":{\"x\":1}}]}", + "{\"timeUnixNano\":\"1\",\"eventName\":\"e\",\"body\":\"s\"}", + "{\"TIMEUNIXNANO\":\"9\",\"BODY\":{\"STRINGVALUE\":\"upper\"}}", + }; + static const char *groups[] = { + "{\"resource\":{\"attributes\":[],\"droppedAttributesCount\":{\"x\":null}}," + "\"schemaUrl\":5,\"scopeLogs\":[{\"schemaUrl\":\"s\",%s}]}", + "{\"resource\":5,\"scopeLogs\":[{\"scope\":5,%s}]}", + "{\"scopeLogs\":[{\"scope\":{\"name\":5,\"version\":\"v\"," + "\"droppedAttributesCount\":null,\"attributes\":[]},%s}]}", + "{\"scopeLogs\":[{\"scope\":{\"attributes\":{}},%s}]}", + "{\"scopeLogs\":[{\"scope\":{\"attributes\":[\"x\"]},%s}]}", + "{\"resource\":{\"attributes\":[1]},\"scopeLogs\":[{%s}]}", + "{\"scopeLogs\":[{\"logRecords\":[]},{\"scope\":{}},{%s}]}", + }; + char inner[1024]; + + for (i = 0; i < sizeof(bodies) / sizeof(bodies[0]); i++) { + snprintf(json, sizeof(json), + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[" + "{\"timeUnixNano\":\"1\",\"body\":%s," + "\"attributes\":[{\"key\":\"a\",\"value\":%s}]}]}]}]}", + bodies[i], bodies[i]); + json_cfl_check_parity(bodies[i], json, SIZE_MAX); + } + for (i = 0; i < sizeof(records) / sizeof(records[0]); i++) { + snprintf(json, sizeof(json), + "{\"resourceLogs\":[{\"scopeLogs\":[{\"logRecords\":[%s]}]}]}", + records[i]); + json_cfl_check_parity(records[i], json, SIZE_MAX); + } + for (i = 0; i < sizeof(groups) / sizeof(groups[0]); i++) { + snprintf(inner, sizeof(inner), groups[i], + "\"logRecords\":[{\"timeUnixNano\":\"1\",\"body\":{\"stringValue\":\"g\"}}]"); + snprintf(json, sizeof(json), "{\"resourceLogs\":[%s]}", inner); + json_cfl_check_parity(groups[i], json, SIZE_MAX); + } +} + +/* Generated requests, see data/opentelemetry/logs_parity_corpus.py */ +void test_opentelemetry_json_cfl_parity_corpus() +{ + int ret; + int count; + char name[32]; + char *corpus; + char *line; + char *end; + size_t size; + size_t limit; + struct flb_log_event_encoder enc; + + corpus = mk_file_to_buffer(OTEL_JSON_PARITY_PATH); + TEST_CHECK(corpus != NULL); + if (corpus == NULL) { + return; + } + + count = 0; + for (line = corpus; *line != '\0'; line = end + 1) { + end = strchr(line, '\n'); + if (end == NULL) { + break; + } + *end = '\0'; + snprintf(name, sizeof(name), "request %d", ++count); + json_cfl_check_parity(name, line, SIZE_MAX); + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + ret = flb_opentelemetry_logs_json_to_msgpack(&enc, line, strlen(line), NULL, NULL); + size = enc.output_length; + flb_log_event_encoder_destroy(&enc); + if (ret == 0) { + for (limit = size > 16 ? size - 16 : 0; limit <= size + 16; limit++) { + json_cfl_check_parity(name, line, limit); + } + } + } + flb_free(corpus); + TEST_CHECK(count > 0); +} + +/* + * The protobuf CFL decoder must produce the records of the msgpack decoder + * and enforce logs_max_size at the same size. The input encodes CFL records + * after processing, the limit only applies to the decoder estimate. + */ +static void protobuf_cfl_check_parity(const char *name, const char *payload, size_t length, + size_t limit) +{ + int ret; + int cfl_ret; + size_t record_count; + struct flb_mp_chunk_cobj *chunk; + struct flb_opentelemetry_logs_protobuf_cfl *result; + struct flb_opentelemetry_logs_decode_options options; + struct flb_log_event_encoder enc; + struct flb_log_event_encoder cfl_enc; + + options.metadata_key = "otlp"; + options.body_key = NULL; + options.output_limit = limit; + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + flb_log_event_encoder_init(&cfl_enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + enc.output_limit = limit; + + ret = flb_opentelemetry_logs_protobuf_to_msgpack(&enc, payload, length, &options, + &record_count); + + result = NULL; + cfl_ret = flb_opentelemetry_logs_protobuf_to_cfl(payload, length, &options, &result); + if (cfl_ret == 0) { + chunk = flb_opentelemetry_logs_protobuf_cfl_take_chunk(result); + chunk->log_encoder = &cfl_enc; + cfl_ret = flb_mp_chunk_cobj_encode_append(chunk); + chunk->log_encoder = NULL; + flb_mp_chunk_cobj_destroy(chunk); + } + flb_opentelemetry_logs_protobuf_cfl_destroy(result); + + TEST_CHECK_((ret == 0) == (cfl_ret == 0), "%s (limit %zu): result %i vs %i", + name, limit, ret, cfl_ret); + if (ret == 0 && cfl_ret == 0) { + TEST_CHECK_(log_events_equal(enc.output_buffer, enc.output_length, + cfl_enc.output_buffer, cfl_enc.output_length), + "%s (limit %zu): records differ", name, limit); + } + + flb_log_event_encoder_destroy(&enc); + flb_log_event_encoder_destroy(&cfl_enc); +} + +void test_opentelemetry_protobuf_cfl_parity_corpus() +{ + int ret; + int count; + char name[32]; + char *payload; + size_t size; + size_t limit; + size_t record_count; + uint32_t length; + unsigned char prefix[4]; + FILE *file; + struct flb_log_event_encoder enc; + struct flb_opentelemetry_logs_decode_options options; + + file = fopen(OTEL_PROTOBUF_PARITY_PATH, "rb"); + TEST_CHECK(file != NULL); + if (file == NULL) { + return; + } + + options.metadata_key = "otlp"; + options.body_key = NULL; + options.output_limit = SIZE_MAX; + count = 0; + + /* each request is prefixed by its little-endian 32-bit length */ + while (fread(prefix, sizeof(prefix), 1, file) == 1) { + length = (uint32_t) prefix[0] | ((uint32_t) prefix[1] << 8) | + ((uint32_t) prefix[2] << 16) | ((uint32_t) prefix[3] << 24); + payload = flb_malloc(length + 1); + TEST_CHECK(payload != NULL); + if (payload == NULL || fread(payload, 1, length, file) != length) { + flb_free(payload); + break; + } + snprintf(name, sizeof(name), "request %d", ++count); + protobuf_cfl_check_parity(name, payload, length, SIZE_MAX); + + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + ret = flb_opentelemetry_logs_protobuf_to_msgpack(&enc, payload, length, &options, + &record_count); + size = enc.output_length; + flb_log_event_encoder_destroy(&enc); + if (ret == 0) { + for (limit = size > 16 ? size - 16 : 0; limit <= size + 16; limit++) { + protobuf_cfl_check_parity(name, payload, length, limit); + } + } + flb_free(payload); + } + fclose(file); + TEST_CHECK(count > 0); +} + +void test_opentelemetry_cases_json_cfl_parity() +{ + int ret; + int type; + char case_name[128]; + char *cases_json; + char *tmp_buf; + char *input_json; + size_t tmp_size; + size_t size; + size_t limit; + size_t i; + size_t input_index; + msgpack_unpacked result; + msgpack_object *root; + msgpack_object *case_obj; + struct flb_log_event_encoder enc; + + cases_json = mk_file_to_buffer(OTEL_TEST_CASES_PATH); + TEST_CHECK(cases_json != NULL); + if (cases_json == NULL) { + return; + } + + ret = flb_pack_json(cases_json, strlen(cases_json), &tmp_buf, &tmp_size, &type, NULL); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_free(cases_json); + return; + } + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, tmp_buf, tmp_size, NULL); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + root = &result.data; + + for (i = 0; i < root->via.map.size; i++) { + case_obj = &root->via.map.ptr[i].val; + + ret = flb_otel_utils_find_map_entry_by_key(&case_obj->via.map, "input", 0, FLB_TRUE); + TEST_CHECK(ret >= 0); + if (ret < 0) { + continue; + } + input_index = ret; + input_json = flb_msgpack_to_json_str(1024, &case_obj->via.map.ptr[input_index].val, + FLB_TRUE); + TEST_CHECK(input_json != NULL); + if (input_json == NULL) { + continue; + } + + snprintf(case_name, sizeof(case_name), "%.*s", + (int) root->via.map.ptr[i].key.via.str.size, + root->via.map.ptr[i].key.via.str.ptr); + json_cfl_check_parity(case_name, input_json, SIZE_MAX); + + /* the logs_max_size boundary must be the same on both paths */ + flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); + ret = flb_opentelemetry_logs_json_to_msgpack(&enc, input_json, strlen(input_json), + NULL, NULL); + size = enc.output_length; + flb_log_event_encoder_destroy(&enc); + if (ret == 0) { + for (limit = size > 64 ? size - 64 : 0; limit <= size + 64; limit++) { + json_cfl_check_parity(case_name, input_json, limit); + } + } + flb_free(input_json); + } + + msgpack_unpacked_destroy(&result); + flb_free(tmp_buf); + flb_free(cases_json); +} + + void test_opentelemetry_traces_cases() { int ret; @@ -4063,6 +4575,13 @@ TEST_LIST = { { "find_map_entry_by_key", test_find_map_entry_by_key }, { "json_payload_get_wrapped_value", test_json_payload_get_wrapped_value }, { "opentelemetry_cases", test_opentelemetry_cases }, + { "opentelemetry_cases_json_cfl_parity", test_opentelemetry_cases_json_cfl_parity }, + { "opentelemetry_json_cfl_parity_values", test_opentelemetry_json_cfl_parity_values }, + { "opentelemetry_json_cfl_lenient_payloads", + test_opentelemetry_json_cfl_lenient_payloads }, + { "opentelemetry_json_cfl_parity_corpus", test_opentelemetry_json_cfl_parity_corpus }, + { "opentelemetry_protobuf_cfl_parity_corpus", + test_opentelemetry_protobuf_cfl_parity_corpus }, { "opentelemetry_logs_otlp_json_roundtrip", test_opentelemetry_logs_otlp_json_roundtrip }, { "opentelemetry_logs_otlp_json_preserves_appended_resources", diff --git a/tests/internal/opentelemetry_protobuf.c b/tests/internal/opentelemetry_protobuf.c index b8e1d4607dd..7b89cbee64d 100644 --- a/tests/internal/opentelemetry_protobuf.c +++ b/tests/internal/opentelemetry_protobuf.c @@ -1,5 +1,5 @@ #include "flb_tests_internal.h" -#include "../../plugins/in_opentelemetry/opentelemetry_protobuf.h" +#include static void test_protobuf_depth(void) { @@ -26,11 +26,11 @@ static void test_protobuf_depth(void) /* 99, 100 and 101 schema messages, including scalar AnyValue leaves. */ size = opentelemetry__proto__common__v1__any_value__pack(&values[1], buffer); - TEST_CHECK(opentelemetry_protobuf_validate(values[1].base.descriptor, buffer, size) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(values[1].base.descriptor, buffer, size) == 0); size = opentelemetry__proto__common__v1__array_value__pack(&arrays[0], buffer); - TEST_CHECK(opentelemetry_protobuf_validate(arrays[0].base.descriptor, buffer, size) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(arrays[0].base.descriptor, buffer, size) == 0); size = opentelemetry__proto__common__v1__any_value__pack(&values[0], buffer); - TEST_CHECK(opentelemetry_protobuf_validate(values[0].base.descriptor, buffer, size) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(values[0].base.descriptor, buffer, size) != 0); } static void test_protobuf_wire_boundaries(void) @@ -45,14 +45,18 @@ static void test_protobuf_wire_boundaries(void) unsigned char fixed[] = {0x21, 0, 0, 0, 0, 0, 0, 0, 0}; descriptor = &opentelemetry__proto__common__v1__any_value__descriptor; - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, NULL, 0) == 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, truncated, sizeof(truncated)) != 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, overflow, sizeof(overflow)) != 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, opaque, sizeof(opaque)) == 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, unknown, sizeof(unknown)) == 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, zero_tag, sizeof(zero_tag)) != 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, fixed, sizeof(fixed)) == 0); - TEST_CHECK(opentelemetry_protobuf_validate(descriptor, fixed, sizeof(fixed) - 1) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, NULL, 0) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, truncated, + sizeof(truncated)) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, overflow, + sizeof(overflow)) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, opaque, sizeof(opaque)) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, unknown, sizeof(unknown)) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, zero_tag, + sizeof(zero_tag)) != 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, fixed, sizeof(fixed)) == 0); + TEST_CHECK(flb_opentelemetry_protobuf_validate(descriptor, fixed, + sizeof(fixed) - 1) != 0); } TEST_LIST = { diff --git a/tests/internal/processor.c b/tests/internal/processor.c index 91824e7f3e3..d0f1135be00 100644 --- a/tests/internal/processor.c +++ b/tests/internal/processor.c @@ -693,6 +693,7 @@ static void processor_metrics_failure_clears_output(void) #define WINDOW_PLAIN_RECORDS 2000 #define WINDOW_TOTAL_RECORDS (WINDOW_GROUPS * WINDOW_GROUP_RECORDS + \ 2 * WINDOW_PLAIN_RECORDS) +#define WINDOW_CORRUPT_RECORDS 3000 static int window_calls; static int window_max_records; @@ -792,6 +793,54 @@ static int create_window_records(char **out_buf, size_t *out_size) return 0; } +static int create_plain_window_records(char **out_buf, size_t *out_size) +{ + int i; + int ret = 0; + struct flb_log_event_encoder *encoder; + + encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); + if (encoder == NULL) { + return -1; + } + + for (i = 0; ret == 0 && i < WINDOW_CORRUPT_RECORDS; i++) { + ret = append_window_record(encoder, i); + } + if (ret != 0) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + *out_buf = flb_malloc(encoder->output_length + 1); + if (*out_buf == NULL) { + flb_log_event_encoder_destroy(encoder); + return -1; + } + + memcpy(*out_buf, encoder->output_buffer, encoder->output_length); + (*out_buf)[encoder->output_length] = (char) 0xc1; + *out_size = encoder->output_length + 1; + flb_log_event_encoder_destroy(encoder); + + return 0; +} + +static int append_corrupt_tail(char **buf, size_t *size) +{ + char *tmp; + + tmp = flb_realloc(*buf, *size + 1); + if (tmp == NULL) { + return -1; + } + + tmp[*size] = (char) 0xc1; + *buf = tmp; + *size += 1; + return 0; +} + static int mark_window_records(struct flb_processor_instance *ins, void *data, const char *tag, int tag_len) { @@ -1107,6 +1156,105 @@ static void processor_log_windows_drop_all(void) window_config_destroy(config); } +static void check_marked_output(void *output, size_t output_size, int expected) +{ + int count = 0; + int type; + struct flb_log_event event; + struct flb_log_event_decoder decoder; + + TEST_ASSERT(flb_log_event_decoder_init(&decoder, output, output_size) == + FLB_EVENT_DECODER_SUCCESS); + + while (flb_log_event_decoder_next(&decoder, &event) == FLB_EVENT_DECODER_SUCCESS) { + TEST_ASSERT(flb_log_event_decoder_get_record_type(&event, &type) == 0); + if (type != FLB_LOG_EVENT_NORMAL) { + continue; + } + TEST_CHECK(event.body->type == MSGPACK_OBJECT_MAP); + TEST_CHECK(event.body->via.map.size == 3); + count++; + } + + TEST_CHECK(count == expected); + flb_log_event_decoder_destroy(&decoder); +} + +static void run_processor_corrupt_tail(int windowed) +{ + int ret; + int records = -1; + int expected; + char *input; + size_t input_size; + void *output = NULL; + size_t output_size = 0; + double errors; + char stage_label[32]; + struct flb_config *config; + struct flb_processor *proc; + struct flb_processor_unit *pu; + struct flb_input_instance owner; + + config = window_config_create(); + TEST_ASSERT(config != NULL); + memset(&owner, 0, sizeof(owner)); + snprintf(owner.name, sizeof(owner.name), "%s", "window_input.0"); + owner.cmt = cmt_create(); + TEST_ASSERT(owner.cmt != NULL); + + if (windowed) { + TEST_ASSERT(create_plain_window_records(&input, &input_size) == 0); + expected = WINDOW_CORRUPT_RECORDS; + proc = window_processor_create(config, &owner, "test_window_mark", NULL); + } + else { + TEST_ASSERT(create_msgpack_records(&input, &input_size) == 0); + TEST_ASSERT(append_corrupt_tail(&input, &input_size) == 0); + expected = 1; + proc = window_processor_create(config, &owner, "test_single_mark", NULL); + } + TEST_ASSERT(proc != NULL); + + window_calls = 0; + ret = flb_processor_run_with_count(proc, 0, FLB_PROCESSOR_LOGS, "test", 4, + input, input_size, &output, &output_size, + -1, &records); + TEST_CHECK(ret == 0); + TEST_CHECK(records == expected); + TEST_CHECK(output != NULL && output != input); + TEST_CHECK(flb_mp_count_log_records(output, output_size) == expected); + if (windowed) { + TEST_CHECK(window_calls > 1); + } + else { + TEST_CHECK(window_calls == 1); + } + check_marked_output(output, output_size, expected); + + pu = mk_list_entry(proc->logs.next, struct flb_processor_unit, _head); + snprintf(stage_label, sizeof(stage_label), "%zu", pu->stage); + TEST_CHECK(get_counter_value_5(proc->cmt_errors, "input", "window_input.0", + (char *) pu->name, stage_label, "logs", &errors) == 0); + TEST_CHECK(errors == 1.0); + + flb_free(output); + flb_free(input); + flb_processor_destroy(proc); + cmt_destroy(owner.cmt); + window_config_destroy(config); +} + +static void processor_corrupt_tail_preserves_prefix(void) +{ + run_processor_corrupt_tail(FLB_FALSE); +} + +static void processor_window_corrupt_tail_preserves_prefix(void) +{ + run_processor_corrupt_tail(FLB_TRUE); +} + static void processor_log_windows_need_every_unit(void) { int i; @@ -1562,6 +1710,9 @@ TEST_LIST = { { "processor_metrics_counters", processor_metrics_counters }, { "processor_log_windows_match_single_chunk", processor_log_windows_match_single_chunk }, { "processor_log_windows_drop_all", processor_log_windows_drop_all }, + { "processor_corrupt_tail_preserves_prefix", processor_corrupt_tail_preserves_prefix }, + { "processor_window_corrupt_tail_preserves_prefix", + processor_window_corrupt_tail_preserves_prefix }, { "processor_log_windows_need_every_unit", processor_log_windows_need_every_unit }, { "processor_chunk_codec_is_per_run", processor_chunk_codec_is_per_run }, { "processor_group_boundaries_follow_the_input", processor_group_boundaries_follow_the_input },