diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 930c8fb8..6a736a01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,9 @@ jobs: uses: actions-rs/cargo@v1 with: command: nextest - args: run --target ${{ matrix.target }} --verbose + # node_metrics_receiver pulls in no extra dependencies, so building it here is + # nearly free and keeps its code and tests compiled on every PR. + args: run --target ${{ matrix.target }} --features node_metrics_receiver --verbose - name: run test processor-sdk uses: actions-rs/cargo@v1 with: @@ -83,8 +85,10 @@ jobs: - name: build (no default features) uses: actions-rs/cargo@v1 with: + # node_metrics_receiver is built here too: it reads /proc and /sys through libc, + # whose statfs field widths differ on 32-bit targets. + args: --target ${{ matrix.target }} --no-default-features --features file_receiver,node_metrics_receiver command: build - args: --target ${{ matrix.target }} --no-default-features --features file_receiver # Check formatting with rustfmt formatting: diff --git a/Cargo.toml b/Cargo.toml index ef0f8bcf..7f2cb121 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ default = ["rdkafka", "aws_iam", "file_receiver", "rust_processor"] fluent_receiver = ["dep:rmpv", "dep:rmp-serde"] file_receiver = ["dep:glob", "dep:notify", "dep:notify-debouncer-mini"] kmsg_receiver = ["dep:tempfile"] +node_metrics_receiver = [] pprof = ["dep:pprof"] pyo3 = ["dep:pyo3", "dep:rotel_python_processor_sdk"] rust_processor = ["dep:rotel_rust_processor_sdk", "dep:abi_stable"] diff --git a/README.md b/README.md index 5c22daf0..c03ab221 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,10 @@ Rotel is ideal for resource-constrained environments and applications where mini - Additional exporters: [ClickHouse](#clickhouse-exporter-configuration), [Datadog](#datadog-exporter-configuration), [AWS X-RAY](#aws-x-ray-exporter-configuration), [AWS EMF](#aws-emf-exporter-configuration), and [Kafka](#kafka-exporter-configuration) -- Additional receivers: [File](#file-receiver-configuration), [Fluent](#fluent-receiver-configuration), [Kafka](#kafka-receiver-configuration), and [Ksmsg](#kmsg-receiver-configuration-linux-only) (Linux kernel messages) +- Additional + receivers: [File](#file-receiver-configuration), [Fluent](#fluent-receiver-configuration), [Kafka](#kafka-receiver-configuration), + [Kmsg](#kmsg-receiver-configuration-linux-only) (Linux kernel messages), and + [Node Metrics](#node-metrics-receiver-configuration) (host CPU, memory, disk, network; Linux-oriented) - [Python](#python-processor-sdk) and [Rust](#rust-processor-sdk) processor SDKs Rotel can be easily bundled with popular runtimes as packages. Its Rust implementation ensures minimal resource usage @@ -120,9 +123,13 @@ curl -X POST http://localhost:4318/v1/traces \ - [Parsers](#parsers) - [Offset Persistence](#offset-persistence) - [Example Usage](#example-usage-1) +- [Node Metrics Receiver](#node-metrics-receiver-configuration) + - [Available Metrics](#available-metrics) + - [Example Usage](#example-usage-2) + - [Testing the Node Metrics Receiver](#testing-the-node-metrics-receiver) - [Kmsg Receiver (Linux-only)](#kmsg-receiver-configuration-linux-only) - [Priority Levels](#priority-levels) - - [Example Usage](#example-usage-2) + - [Example Usage](#example-usage-3) - [Testing the Kmsg Receiver](#testing-the-kmsg-receiver) - [Log Record Format](#log-record-format) - [Batch configuration](#batch-configuration) @@ -963,6 +970,480 @@ rotel start \ --otlp-exporter-endpoint "localhost:4317" ``` +### Node Metrics Receiver configuration + +**NOTE**: The Node Metrics Receiver is currently only included when built with the opt-in feature +`--features node_metrics_receiver`. + +The Node Metrics Receiver periodically scrapes system metrics from the host machine and converts them to OpenTelemetry +metrics format. It reads directly from Linux kernel interfaces (`/proc`, `/sys`, syscalls), emitting raw counter values +compatible with Prometheus node_exporter conventions. + +**NOTE**: This receiver targets Linux and requires access to the `/proc` and `/sys` filesystems. Elsewhere only the +`uname`, `time` and `textfile` collectors return data — the collectors that read `/proc` or `/sys` yield nothing — and a +warning is logged at startup. + +Metric names, help strings and the default filesystem-type exclusion list follow Prometheus +[node_exporter](https://github.com/prometheus/node_exporter) (Apache-2.0), so existing dashboards and alerting rules +largely carry over. The collector split differs in one place: `node_boot_time_seconds` is produced by the `time` +collector here, while node_exporter exposes it from its `stat` collector. + +Select the Node Metrics receiver with the option `--receiver node_metrics`. + +| Option | Default | Description | +|--------------------------------------------------|--------------|------------------------------------------------------------------------------------------| +| --node-metrics-receiver-scrape-interval | 60s | Scrape interval as a string time duration (minimum 1 second) | +| --node-metrics-receiver-cpu | true | Enable CPU metrics collection | +| --node-metrics-receiver-loadavg | true | Enable load average metrics collection | +| --node-metrics-receiver-memory | true | Enable memory metrics collection | +| --node-metrics-receiver-network | true | Enable network metrics collection | +| --node-metrics-receiver-filesystem | true | Enable filesystem metrics collection | +| --node-metrics-receiver-uname | true | Enable system info metrics (kernel, hostname) | +| --node-metrics-receiver-stat | true | Enable kernel stat counters (forks, context switches, interrupts, procs running/blocked) | +| --node-metrics-receiver-processes | true | Enable process metrics (kernel limits: threads-max, pid_max) | +| --node-metrics-receiver-diskstats | true | Enable disk I/O statistics (Linux only, /proc/diskstats) | +| --node-metrics-receiver-vmstat | true | Enable virtual memory statistics (/proc/vmstat) | +| --node-metrics-receiver-netstat | true | Enable network statistics (/proc/net/netstat and /proc/net/snmp) | +| --node-metrics-receiver-sockstat | true | Enable socket statistics (/proc/net/sockstat and /proc/net/sockstat6) | +| --node-metrics-receiver-filefd | true | Enable file descriptor statistics (/proc/sys/fs/file-nr) | +| --node-metrics-receiver-cpufreq | true | Enable CPU frequency metrics (/sys/devices/system/cpu/cpu*/cpufreq/) | +| --node-metrics-receiver-thermal-zone | true | Enable thermal zone and cooling device metrics (/sys/class/thermal/) | +| --node-metrics-receiver-nvme | true | Enable NVMe device info metrics (/sys/class/nvme/) | +| --node-metrics-receiver-hwmon | true | Enable hardware monitoring sensor metrics (/sys/class/hwmon/) | +| --node-metrics-receiver-time | true | Enable time metrics (node_time_seconds, node_boot_time_seconds) | +| --node-metrics-receiver-textfile | false | Enable textfile collector for custom Prometheus metrics | +| --node-metrics-receiver-textfile-directory | | Directory of Prometheus-format textfiles (*.prom), or a single .prom file | +| --node-metrics-receiver-filesystem-mount-exclude | | Regex pattern to exclude filesystem mount points, in addition to the built-in excludes | +| --node-metrics-receiver-include-filter | | Regex pattern to include only matching metric names | +| --node-metrics-receiver-exclude-filter | | Regex pattern to exclude matching metric names | +| --node-metrics-receiver-rootfs-path | / | Prefix under which the host root is mounted (for containerized monitoring) | +| --node-metrics-receiver-procfs-path | /proc | Path to procfs mount point (for containerized monitoring) | +| --node-metrics-receiver-sysfs-path | /sys | Path to sysfs mount point (for containerized monitoring) | +| --node-metrics-receiver-service-name | node_metrics | Service name for the OTLP Resource attribute | + +The collector toggles are booleans that take an explicit value, for example `--node-metrics-receiver-cpu false` or +`--node-metrics-receiver-cpu=true`. Passing one of them without a value is an error. The same applies to the +environment variable form, `ROTEL_NODE_METRICS_RECEIVER_CPU=false`. + +`--node-metrics-receiver-rootfs-path` is prefixed onto the procfs and sysfs paths, so `/host` alone yields +`/host/proc` and `/host/sys`. A procfs or sysfs path that is already *under* the rootfs prefix is rejected at startup +with an error naming both flags, rather than silently producing `/host/host/proc` — so pass either the rootfs path or +the already-prefixed individual paths, not both. The comparison is path-component aware, so only a genuine prefix is +rejected: `/host` with `/host/proc` is an error, while `/host` with `/hostile/proc` or `/proc-alt` is accepted and +composes normally into `/host/hostile/proc` and `/host/proc-alt`. The rootfs path applies to the procfs and sysfs paths +only, so `--node-metrics-receiver-textfile-directory` is always used exactly as given. + +A few more startup and runtime behaviours: + +- The scrape interval is a string time duration, so `30s`, `90s` or `2m`, with a 1 second minimum enforced at startup. +- The first scrape runs immediately at startup rather than after one interval. Ticks stay aligned to the original + schedule, so a tick missed while a slow scrape was in progress is skipped rather than delayed or fired back-to-back + as a catch-up burst. +- A scrape that runs longer than the scrape interval, or 30 seconds if the interval is shorter than that, is timed out, + logged, and skipped. (The floor matters: with a 1 second interval, a merely slow host would otherwise time out on + every cycle and the receiver would report nothing at all.) Because a blocking read cannot be aborted, scraping resumes + only once the timed-out scrape actually finishes; each interval until then is skipped and counted as a scrape failure. + An unresponsive filesystem is the usual cause — exclude it with + `--node-metrics-receiver-filesystem-mount-exclude`. +- A collector that panics does not take the receiver down: the panic is logged, counted as a scrape failure, and + scraping continues on the next tick. +- A slow or full pipeline does not block scraping. If the pipeline has not accepted a batch within 10 seconds (or the + scrape interval, whichever is shorter), the batch is abandoned with a warning and counted as refused, and the next + tick scrapes as usual. The warning notes that the batch "may still be delivered", because the hand-off can complete + after the timeout has fired. Without this bound a backed-up exporter would park the receiver indefinitely. +- If no configured exporter accepts metrics, the receiver logs a warning and stays idle without scraping. +- Enabling `--node-metrics-receiver-textfile` without `--node-metrics-receiver-textfile-directory` is a startup error. + Setting the directory without enabling the collector only logs a warning. +- Duplicate entries in the collector list are removed, keeping the first occurrence. This matters only when the config + is built programmatically; the CLI toggles cannot name a collector twice. +- Within one metric, data points that repeat an already seen label set are dropped with a warning and the first + occurrence is kept, since duplicates would break the OTLP single-writer principle. +- Every batch carries the resource attributes `service.name` (from `--node-metrics-receiver-service-name`) and, on + Linux, `os.type`. `host.name` (the local hostname) is added too, and is what keeps otherwise identical series from + different hosts distinct at the backend — it is omitted only if the hostname is not valid UTF-8. +- On shutdown, a batch already in flight is given a short grace window to reach the pipeline; if the window expires, or + the pipeline has already closed, the batch is reported as refused. A scrape still running when cancellation arrives is + discarded without being sent. +- With [internal telemetry](#internal-telemetry) enabled, the receiver reports + `rotel_receiver_accepted_metric_points`, `rotel_receiver_refused_metric_points`, `rotel_receiver_scrape_failures` and + `rotel_receiver_empty_scrapes`, all tagged `receiver="node_metrics"`. Refused metric points are the points of a batch + that could not be handed to the pipeline: the pipeline was closed, the shutdown grace window expired, or the send + timed out. A scrape failure is counted when a scrape exceeds the interval and is timed out, when a collector panics, + and for every cycle skipped while a previous scrape is still running. A scrape that collects nothing at all + increments `rotel_receiver_empty_scrapes` and is logged as a warning; it is not counted as a scrape failure, since an + empty result is a configuration problem rather than a failed scrape. + +#### Available Metrics + +The receiver collects the following metrics (names follow Prometheus node_exporter conventions): + +**CPU** (when `--node-metrics-receiver-cpu=true`): +- `node_cpu_seconds_total` - Seconds the CPUs spent in each mode (with `cpu` and `mode` labels: user, nice, system, idle, iowait, irq, softirq, steal) +- `node_cpu_guest_seconds_total` - Seconds the CPUs spent in guest mode (with `cpu` and `mode` labels: user, nice) + +**NOTE**: Only per-core lines from `/proc/stat` are reported. The aggregate `cpu` line is deliberately skipped, as +Prometheus node_exporter does, so there is no all-core total metric; sum across the `cpu` label instead. + +**Load Average** (when `--node-metrics-receiver-loadavg=true`): +- `node_load1` - 1-minute load average +- `node_load5` - 5-minute load average +- `node_load15` - 15-minute load average + +**Memory** (when `--node-metrics-receiver-memory=true`): +All fields from `/proc/meminfo` are exposed dynamically, including: +- `node_memory_MemTotal_bytes` - Total memory in bytes +- `node_memory_MemFree_bytes` - Free memory in bytes +- `node_memory_MemAvailable_bytes` - Available memory in bytes +- `node_memory_Buffers_bytes` - Buffer memory in bytes +- `node_memory_Cached_bytes` - Cached memory in bytes +- `node_memory_SwapTotal_bytes` - Total swap in bytes +- `node_memory_SwapFree_bytes` - Free swap in bytes +- ... and 30+ additional memory metrics + +**NOTE**: Only fields carrying the `kB` suffix are converted to bytes; those get the `_bytes` name suffix and the unit +`By`. Unitless fields keep their kernel name and carry no unit, for example `HugePages_Total` is exposed as +`node_memory_HugePages_Total`. Parenthesised field names are normalised, so `Active(anon)` becomes +`node_memory_Active_anon_bytes`. + +**Network** (when `--node-metrics-receiver-network=true`, per interface with `device` label): +- `node_network_receive_bytes_total` - Bytes received +- `node_network_receive_packets_total` - Packets received +- `node_network_receive_errs_total` - Receive errors +- `node_network_receive_drop_total` - Receive drops +- `node_network_receive_fifo_total` - Receive FIFO errors +- `node_network_receive_frame_total` - Receive frame errors +- `node_network_receive_compressed_total` - Compressed packets received +- `node_network_receive_multicast_total` - Multicast packets received +- `node_network_transmit_bytes_total` - Bytes transmitted +- `node_network_transmit_packets_total` - Packets transmitted +- `node_network_transmit_errs_total` - Transmit errors +- `node_network_transmit_drop_total` - Transmit drops +- `node_network_transmit_fifo_total` - Transmit FIFO errors +- `node_network_transmit_colls_total` - Transmit collisions +- `node_network_transmit_carrier_total` - Transmit carrier errors +- `node_network_transmit_compressed_total` - Compressed packets transmitted + +**Filesystem** (when `--node-metrics-receiver-filesystem=true`, per mount with `device`, `fstype`, `mountpoint` labels): +- `node_filesystem_size_bytes` - Total filesystem size +- `node_filesystem_free_bytes` - Free space (total free blocks) +- `node_filesystem_avail_bytes` - Available space (free blocks for non-root users) +- `node_filesystem_files` - Total file nodes (inodes) +- `node_filesystem_files_free` - Free file nodes +- `node_filesystem_readonly` - Whether the filesystem is mounted read-only (1 = readonly) + +**NOTE**: +- Virtual filesystems are automatically filtered out. Besides the obvious ones (proc, sysfs, cgroup, overlay), the list + also covers `squashfs`, `iso9660` and `autofs` — so snap and other squashfs-backed mounts, and CD/DVD images, are not + reported. +- Network filesystem types are also excluded by default, because a `statfs` call on an unreachable server can block or + hang. Examples include NFS, SMB/CIFS, Ceph, GlusterFS, Lustre, BeeGFS, GPFS, 9p, AFS, NCP, WebDAV, and remote FUSE + mounts such as sshfs and s3fs; the authoritative list is `collect_filesystem` in + `src/receivers/node_metrics/collector/procfs.rs`. Local FUSE filesystems, such as mergerfs or gocryptfs, are reported + normally. This list of excluded types is not currently overridable: + `--node-metrics-receiver-filesystem-mount-exclude` can only remove further mount points, it cannot bring an excluded + type back. +- Mount points are excluded by prefix, with the match respecting path boundaries, so `/dev` does not exclude + `/developer`. `/proc`, `/sys` and `/dev` are excluded including the mount point itself. `/run/credentials`, + `/run/user`, `/var/lib/docker` and `/var/lib/containers` are excluded only *below* the path: a filesystem mounted + exactly at one of them is a real volume and is reported, which matters most for a dedicated `/var/lib/docker` disk. +- Duplicate mounts (same device + mountpoint) are deduplicated. +- Mount paths containing octal escapes (e.g. `\040` for spaces) are decoded automatically. + +**System/Uname** (when `--node-metrics-receiver-uname=true`): +- `node_uname_info` - System info gauge (labels: sysname, release, version, nodename, machine, domainname [Linux only]) + +**Time** (when `--node-metrics-receiver-time=true`): +- `node_boot_time_seconds` - System boot time in seconds since epoch (from the `btime` line of `/proc/stat`, read at + startup and re-read on every scrape that reads `/proc/stat` — that is, whenever the `cpu`, `stat` or `time` collector + is enabled; the metric is omitted, rather than reported as zero, while the boot time is unknown, and a value that is + not plausible — zero, negative, or in the future — is rejected rather than published) +- `node_time_seconds` - Current system time in seconds since epoch + +**NOTE**: The boot time is not latched, because the kernel's `btime` moves whenever the wall clock is stepped — a device +with no RTC boots near the epoch and then jumps once NTP syncs. `node_boot_time_seconds` therefore tracks the current +kernel value, while the start time stamped on cumulative counters stays latched at the first known value, so a clock +step is not reported downstream as a counter reset. + +**Stat** (when `--node-metrics-receiver-stat=true`): +- `node_forks_total` - Total number of forks since boot +- `node_context_switches_total` - Total number of context switches +- `node_intr_total` - Total number of interrupts serviced (unit `{interrupts}`) +- `node_procs_running` - Number of processes in runnable state +- `node_procs_blocked` - Number of processes blocked waiting for I/O + +**Processes** (when `--node-metrics-receiver-processes=true`): +- `node_processes_max_threads` - Kernel thread limit (`/proc/sys/kernel/threads-max`) +- `node_processes_max_processes` - Kernel PID limit (`/proc/sys/kernel/pid_max`) + +**Disk I/O** (when `--node-metrics-receiver-diskstats=true`, per device with `device` label): +- `node_disk_reads_completed_total` - Total number of reads completed +- `node_disk_reads_merged_total` - Total number of reads merged +- `node_disk_read_bytes_total` - Total bytes read +- `node_disk_read_time_seconds_total` - Total time spent reading +- `node_disk_writes_completed_total` - Total number of writes completed +- `node_disk_writes_merged_total` - Total number of writes merged +- `node_disk_written_bytes_total` - Total bytes written +- `node_disk_write_time_seconds_total` - Total time spent writing +- `node_disk_io_now` - Number of I/Os currently in progress +- `node_disk_io_time_seconds_total` - Total time spent doing I/Os +- `node_disk_io_time_weighted_seconds_total` - Weighted time spent doing I/Os +- `node_disk_discards_completed_total` - Total discards completed (kernel 4.18+) +- `node_disk_discards_merged_total` - Total discards merged (kernel 4.18+) +- `node_disk_discarded_sectors_total` - Total sectors discarded (kernel 4.18+) +- `node_disk_discard_time_seconds_total` - Total time spent discarding (kernel 4.18+) +- `node_disk_flush_requests_total` - Total flush requests completed (kernel 5.5+) +- `node_disk_flush_requests_time_seconds_total` - Total time spent flushing (kernel 5.5+) + +**NOTE**: Virtual devices (ram, zram, loop, fd) and partitions are automatically filtered out. + +**Vmstat** (when `--node-metrics-receiver-vmstat=true`): +All fields from `/proc/vmstat` are exposed dynamically as `node_vmstat_{field}`, including: +- `node_vmstat_pgfault` - Page faults +- `node_vmstat_pgmajfault` - Major page faults +- `node_vmstat_pgpgin` - Pages paged in +- `node_vmstat_pgpgout` - Pages paged out +- `node_vmstat_pswpin` - Pages swapped in +- `node_vmstat_pswpout` - Pages swapped out +- ... and 100+ additional vmstat metrics + +**NOTE**: The vmstat collector emits all fields from `/proc/vmstat`, which can be 100+ metrics on modern kernels. +To reduce cardinality, either disable the collector entirely (`--node-metrics-receiver-vmstat false`) +or use an include filter that lists all desired metric prefixes across all collectors, e.g.: +`--node-metrics-receiver-include-filter "^node_(vmstat_(oom_kill|pgpg|pswp|pg.*fault)|load|memory_|cpu_)"` + +**Netstat** (when `--node-metrics-receiver-netstat=true`): +TCP/IP statistics from `/proc/net/netstat` and `/proc/net/snmp` as `node_netstat_{Protocol}_{Field}`: +- `node_netstat_TcpExt_TCPTimeouts` - TCP timeouts +- `node_netstat_TcpExt_TCPRetransFail` - TCP retransmit failures +- `node_netstat_TcpExt_SyncookiesSent` - SYN cookies sent +- `node_netstat_IpExt_InOctets` - IP octets received +- `node_netstat_IpExt_OutOctets` - IP octets sent +- `node_netstat_Tcp_CurrEstab` - TCP connections currently established (from /proc/net/snmp) +- ... and many more TCP/IP statistics + +**NOTE**: Like vmstat, the netstat collector emits every field of both files, which is 200+ metrics — considerably more +than node_exporter, which whitelists a subset by default. On a backend billed per series, either disable the collector +or narrow it with an include filter such as +`--node-metrics-receiver-include-filter "^node_netstat_(Tcp|Udp)_"`. + +**Sockstat** (when `--node-metrics-receiver-sockstat=true`): +Socket allocation statistics from `/proc/net/sockstat` and `/proc/net/sockstat6`: +- `node_sockstat_sockets_used` - Total sockets in use +- `node_sockstat_TCP_inuse` - TCP sockets in use +- `node_sockstat_TCP_orphan` - TCP orphaned sockets +- `node_sockstat_TCP_tw` - TCP TIME_WAIT sockets +- `node_sockstat_TCP_alloc` - TCP sockets allocated +- `node_sockstat_TCP_mem` - TCP memory pages +- `node_sockstat_TCP_mem_bytes` - TCP memory in bytes +- `node_sockstat_UDP_inuse` - UDP sockets in use +- `node_sockstat_UDP_mem` - UDP memory pages +- `node_sockstat_UDP_mem_bytes` - UDP memory in bytes +- `node_sockstat_RAW_inuse` - RAW sockets in use +- `node_sockstat_TCP6_inuse` - TCP6 sockets in use (from sockstat6) +- ... and additional IPv6 socket statistics + +**Filefd** (when `--node-metrics-receiver-filefd=true`): +- `node_filefd_allocated` - Number of allocated file descriptors (unit `{file_descriptors}`) +- `node_filefd_maximum` - Maximum number of file descriptors allowed (unit `{file_descriptors}`) + +**CPU Frequency** (when `--node-metrics-receiver-cpufreq=true`, read from +`/sys/devices/system/cpu/cpu*/cpufreq/`, per CPU with `cpu` label): +- `node_cpu_scaling_frequency_hertz` - Current CPU scaling frequency +- `node_cpu_scaling_frequency_min_hertz` - Minimum CPU scaling frequency +- `node_cpu_scaling_frequency_max_hertz` - Maximum CPU scaling frequency +- `node_cpu_frequency_hertz` - Current CPU frequency (from cpuinfo) +- `node_cpu_frequency_min_hertz` - Minimum CPU frequency (from cpuinfo) +- `node_cpu_frequency_max_hertz` - Maximum CPU frequency (from cpuinfo) + +**NOTE**: Only directories named `cpu` followed by digits (`cpu0`, `cpu1`, ...) are scanned, so entries such as +`cpuidle` and `cpufreq` under the same parent are ignored. + +**Thermal Zone** (when `--node-metrics-receiver-thermal-zone=true`): +- `node_thermal_zone_temp` - Thermal zone temperature in Celsius (with `zone` and `type` labels) +- `node_cooling_device_cur_state` - Current cooling device state (with `name` and `type` labels) +- `node_cooling_device_max_state` - Maximum cooling device state (with `name` and `type` labels) + +**NVMe** (when `--node-metrics-receiver-nvme=true`): +- `node_nvme_info` - NVMe device info gauge (with `device`, `firmware_revision`, `model`, `serial`, `state` labels) + +**NOTE**: Only entries named `nvme` followed by digits (`nvme0`, `nvme1`, ...) are scanned, so any other entry under +`/sys/class/nvme` is ignored. + +**Hwmon** (when `--node-metrics-receiver-hwmon=true`, sensor readings carry `chip` and `sensor` labels): +- `node_hwmon_temp_celsius` - Temperature sensor reading +- `node_hwmon_temp_max_celsius` - Temperature max threshold +- `node_hwmon_temp_crit_celsius` - Temperature critical threshold +- `node_hwmon_in_volts` - Voltage sensor reading +- `node_hwmon_fan_rpm` - Fan speed, in revolutions per minute (unit `{rev}/min`) +- `node_hwmon_power_watts` - Power consumption in watts +- `node_hwmon_curr_amps` - Current in amps (converted from milliamps) +- `node_hwmon_chip_names` - Annotation gauge carrying the human-readable chip model (with `chip` and `chip_name` labels) +- `node_hwmon_sensor_label` - Annotation gauge carrying the human-readable sensor label, when the chip provides one (with `chip`, `sensor` and `label` labels) + +**NOTE**: The `sensor` label is the raw sysfs name (`temp1`, `in0`), matching Prometheus node_exporter — it is unique +within a chip, so two sensors can never collapse onto one series, and it does not move when a driver gains or changes a +label file. Where the chip provides human-readable text, that is published separately by `node_hwmon_sensor_label` and +can be joined on `chip` and `sensor`. + +**NOTE**: The `chip` label is derived from the backing device of the hwmon instance, so two chips reporting the same +model name (for example two NVMe drives, both named `nvme`) do not collapse onto the same label set. The model name +itself is published separately by `node_hwmon_chip_names`, which can be joined on `chip`. When an hwmon instance has no +backing `device`, the identity falls back to the chip model name combined with the `hwmonN` directory, which is still +unique within a host but is not stable across reboots, since the hwmon index depends on module load order. + +**NOTE**: The `sensor` label uses the human-readable name from the sysfs `*_label` file when available +(e.g., "Core 0", "Package id 0"), falling back to the raw sensor name (e.g., "temp1"). + +**NOTE**: Only the temperature, voltage, fan, power and current sensor kinds are read. Any other kind exposed by a chip +is skipped with a debug log. + +**Textfile** (when `--node-metrics-receiver-textfile=true`): +Custom metrics from Prometheus-format text files. `--node-metrics-receiver-textfile-directory` accepts either a +directory, in which case every `*.prom` file in it is read in sorted filename order, or the path of a single `.prom` +file. The directory scan is not recursive, so subdirectories are never descended into, and an entry that is itself a +directory named `something.prom` is skipped silently. Symlinks are followed, so a symlinked `.prom` file is collected +like any other. +The `--node-metrics-receiver-include-filter` and `--node-metrics-receiver-exclude-filter` patterns apply to textfile +metrics just as they do to the built-in ones. Accepted input: +- Metric lines: `metric_name{label="value"} 42.5` +- TYPE comments: `# TYPE metric_name counter|gauge` (used to set the metric type) +- HELP comments: `# HELP metric_name Description` (propagated as the OTLP metric description) + +Support for the exposition format is deliberately partial: +- Only the `counter` and `gauge` types are supported. `histogram`, `summary` and `untyped` are treated as gauges, as is + any metric with no `# TYPE` line. +- Trailing per-sample timestamps are accepted but ignored; every data point is stamped with the scrape time. +- `# TYPE` and `# HELP` are scoped to the file they appear in, so the same metric name can be declared differently in + two files. Conflicting `# TYPE` declarations produce two separate OTLP metrics sharing that name, one gauge and one + sum, because metrics are grouped by name *and* type. Within each of those groups the unit and description of the first + sample seen win. +- A textfile larger than 10 MiB is skipped, guarding against a runaway writer script, and its + `node_textfile_scrape_error` is set to 1. The limit is enforced while reading, not from the file's reported size, so a + file that grows between the two cannot slip past it. Its `node_textfile_mtime_seconds` is still emitted even though the + file itself is skipped, so a writer that has stalled stays observable. +- At most 100,000 samples are taken from the textfile directory per scrape. Beyond that the remainder of the file is + dropped, a warning names how many samples were kept, and no further files are read — a bound on memory that the + per-file size limit alone does not give, since a small file can expand into a great many samples. +- Files that are not regular files are skipped silently: a FIFO named `*.prom` would otherwise block the scrape + indefinitely when opened. A symlink to a regular file is followed and collected; a `.prom` entry that cannot be read + at all, such as a dangling symlink, sets `node_textfile_scrape_error` to 1. +- A line with a malformed label — an invalid or empty label name, or a value that is not `key="value"` — is rejected in + full and counted as a malformed line, rather than being emitted without that label. Dropping a label would change the + series identity and silently merge samples that are meant to be distinct. A label name repeated within one line keeps + its first value. + +Additionally emits: +- `node_textfile_scrape_error` - Per-file error indicator (0 = success, 1 = the file could not be read, exceeded the + 10 MiB size limit, or contained a malformed line; with `filename` label). A textfile directory that is missing or + cannot be read yields a single error point whose `filename` label holds the configured path rather than a basename. +- `node_textfile_mtime_seconds` - Modification time of the file in seconds since epoch (with `filename` label), useful for spotting a writer script that has stopped updating + +Example textfile (`/var/lib/node_exporter/textfile/custom.prom`): +``` +# HELP my_app_version Application version info +# TYPE my_app_version gauge +my_app_version{version="1.2.3"} 1 +# TYPE my_app_requests_total counter +my_app_requests_total{endpoint="/api"} 12345 +``` + +#### Example Usage + +Basic usage collecting all metrics and exporting to OTLP: + +```shell +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-scrape-interval 30s \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +Collecting only CPU and memory metrics: + +```shell +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-loadavg false \ + --node-metrics-receiver-network false \ + --node-metrics-receiver-filesystem false \ + --node-metrics-receiver-uname false \ + --node-metrics-receiver-stat false \ + --node-metrics-receiver-processes false \ + --node-metrics-receiver-diskstats false \ + --node-metrics-receiver-vmstat false \ + --node-metrics-receiver-netstat false \ + --node-metrics-receiver-sockstat false \ + --node-metrics-receiver-filefd false \ + --node-metrics-receiver-cpufreq false \ + --node-metrics-receiver-thermal-zone false \ + --node-metrics-receiver-nvme false \ + --node-metrics-receiver-hwmon false \ + --node-metrics-receiver-time false \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +Using filters to only collect memory-related metrics: + +```shell +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-include-filter "^node_memory_" \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +Monitoring the host system from inside a container. Mount the host root, and point the receiver at it with a single +rootfs path: + +```shell +# docker run -v /:/host:ro,rslave ... +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-rootfs-path /host \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +**NOTE**: mounting only `/proc` and `/sys` is enough for every collector *except* `filesystem`. Filesystem metrics need +the host root as well, because the mount table names host paths that `statfs` has to be able to resolve — the rootfs +path is what makes that work. Without it the receiver would report the container's own mounts labelled as though they +were the host's, so prefer the form above over setting `--node-metrics-receiver-procfs-path` and +`--node-metrics-receiver-sysfs-path` individually. The mount table is read from PID 1 (`/proc/1/mounts`) rather than +`/proc/self/mounts` for the same reason: the latter is always the reading process's own mount namespace, whatever +procfs it is read from. + +**NOTE**: The same reasoning applies to a hardened systemd unit on the host. With `PrivateMounts=`, `PrivateTmp=` or +`ProtectSystem=`, rotel's mount namespace differs from PID 1's: the mount table names the host's paths while `statfs` +resolves inside the unit's namespace, so a metric labelled `mountpoint="/tmp"` would describe the unit's private tmpfs. +Prometheus `node_exporter` behaves the same way. Run the receiver without mount-namespace isolation if filesystem +metrics matter. + +Using the textfile collector for custom Prometheus metrics: + +```shell +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-textfile true \ + --node-metrics-receiver-textfile-directory /var/lib/node_exporter/textfile \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +#### Testing the Node Metrics Receiver + +To run integration tests that read the real `/proc` and `/sys` of a Linux host: + +```shell +NODE_METRICS_INTEGRATION_TESTS=true cargo test --test node_metrics_integration_tests --features node_metrics_receiver +``` + +The tests are skipped unless `NODE_METRICS_INTEGRATION_TESTS=true` is set and the `node_metrics_receiver` feature is +enabled. Collectors for optional hardware (thermal zones, hwmon sensors, NVMe devices, cpufreq, diskstats) are absent in +many containers and virtual machines, so the tests for those accept an empty result and require only that any value +produced is finite — asserting a particular piece of hardware is present would make them flaky by design. + ### Kmsg Receiver configuration (Linux-only) **NOTE**: The Kmsg Receiver is Linux-only and must be enabled with the feature flag `--features kmsg_receiver`. @@ -1153,10 +1634,10 @@ to receive data via OTLP and consume from Kafka topics at the same time. The following configuration parameters enable multiple receivers: -| Option | Default | Options | -|-------------|---------|---------------------------------------------| -| --receiver | otlp | otlp, kafka, fluent, file, kmsg | -| --receivers | | comma-separated list (otlp,kafka,file,kmsg) | +| Option | Default | Options | +|-------------|---------|----------------------------------------------------------| +| --receiver | otlp | otlp, kafka, fluent, file, kmsg, node_metrics | +| --receivers | | comma-separated list (otlp,kafka,file,kmsg,node_metrics) | **Important Notes:** diff --git a/build.rs b/build.rs index 1dc44f0f..eaae52f7 100644 --- a/build.rs +++ b/build.rs @@ -3,9 +3,11 @@ use std::io::Result; fn main() -> Result<()> { println!("cargo::rustc-check-cfg=cfg(kafka_integration_tests, values(\"true\"))"); println!("cargo::rustc-check-cfg=cfg(kmsg_integration_tests, values(\"true\"))"); + println!("cargo::rustc-check-cfg=cfg(node_metrics_integration_tests, values(\"true\"))"); println!("cargo::rustc-check-cfg=cfg(rust_processor_integration_tests, values(\"true\"))"); println!("cargo::rerun-if-env-changed=KAFKA_INTEGRATION_TESTS"); println!("cargo::rerun-if-env-changed=KMSG_INTEGRATION_TESTS"); + println!("cargo::rerun-if-env-changed=NODE_METRICS_INTEGRATION_TESTS"); println!("cargo::rerun-if-env-changed=RUST_PROCESSOR_INTEGRATION_TESTS"); println!("cargo::rerun-if-changed=proto/datadog"); prost_build::compile_protos(&["proto/datadog/agent_payload.proto"], &["proto/datadog"])?; @@ -16,6 +18,9 @@ fn main() -> Result<()> { let env = std::env::var("KMSG_INTEGRATION_TESTS").unwrap_or("false".to_string()); println!("cargo:rustc-cfg=kmsg_integration_tests=\"{}\"", env); + let env = std::env::var("NODE_METRICS_INTEGRATION_TESTS").unwrap_or("false".to_string()); + println!("cargo:rustc-cfg=node_metrics_integration_tests=\"{}\"", env); + let env = std::env::var("RUST_PROCESSOR_INTEGRATION_TESTS").unwrap_or("false".to_string()); println!( "cargo:rustc-cfg=rust_processor_integration_tests=\"{}\"", diff --git a/src/init/activation.rs b/src/init/activation.rs index a2797ef7..d4967e7b 100644 --- a/src/init/activation.rs +++ b/src/init/activation.rs @@ -72,6 +72,8 @@ fn all_traces_receivers_disabled(rc: &HashMap) -> bool ReceiverConfig::File(_) => {} // File receiver doesn't handle traces #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] ReceiverConfig::Kmsg(_) => {} // Kmsg receiver doesn't handle traces + #[cfg(feature = "node_metrics_receiver")] + ReceiverConfig::NodeMetrics(_) => {} // Node metrics receiver doesn't handle traces } } true @@ -98,6 +100,8 @@ fn all_metrics_receivers_disabled(rc: &HashMap) -> boo ReceiverConfig::File(_) => {} // File receiver doesn't handle metrics #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] ReceiverConfig::Kmsg(_) => {} // Kmsg receiver doesn't handle metrics + #[cfg(feature = "node_metrics_receiver")] + ReceiverConfig::NodeMetrics(_) => return false, // Node metrics receiver handles metrics } } true @@ -123,7 +127,78 @@ fn all_logs_receivers_disabled(rc: &HashMap) -> bool { ReceiverConfig::File(_) => return false, // File receiver handles logs #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] ReceiverConfig::Kmsg(_) => return false, // Kmsg receiver handles logs + #[cfg(feature = "node_metrics_receiver")] + ReceiverConfig::NodeMetrics(_) => {} // Node metrics receiver doesn't handle logs } } true } + +#[cfg(all(test, feature = "node_metrics_receiver"))] +mod tests { + use super::*; + use crate::init::config::ExporterConfig; + use crate::receivers::node_metrics::config::NodeMetricsConfig; + + /// A receiver map holding only the node metrics receiver + fn node_metrics_only() -> HashMap { + HashMap::from([( + Receiver::NodeMetrics, + ReceiverConfig::NodeMetrics(NodeMetricsConfig::default()), + )]) + } + + /// The node metrics receiver is the only source of metrics in its own pipeline. + /// + /// This single match arm decides whether the metrics pipeline is activated at all: + /// folded in with the neighbouring receivers that do not produce metrics, the + /// receiver would scrape forever into a pipeline that was never started. + #[test] + fn test_node_metrics_receiver_keeps_the_metrics_pipeline_enabled() { + let receivers = node_metrics_only(); + + assert!( + !all_metrics_receivers_disabled(&receivers), + "a node metrics receiver must keep the metrics pipeline enabled" + ); + + // The `false` above has to come from the node metrics arm, not from an + // unconditionally false result: with no receivers at all, metrics stay disabled. + assert!(all_metrics_receivers_disabled(&HashMap::new())); + + // It produces neither traces nor logs, so those pipelines stay disabled — the + // arms that would activate a whole pipeline for nothing. + assert!(all_traces_receivers_disabled(&receivers)); + assert!(all_logs_receivers_disabled(&receivers)); + } + + /// The same decision as seen by the agent, which acts on `TelemetryActivation`. + #[test] + fn test_activation_with_only_a_node_metrics_receiver() { + let exporters = ExporterConfigs { + metrics: vec![ExporterConfig::Blackhole], + ..Default::default() + }; + + let activation = TelemetryActivation::from_config(&node_metrics_only(), &exporters, false); + + // Metrics are the only pipeline this receiver feeds: it has both a receiver and + // an exporter, so it is the only one that comes up. + assert_eq!(activation.metrics, TelemetryState::Active); + assert_eq!(activation.traces, TelemetryState::Disabled); + assert_eq!(activation.logs, TelemetryState::Disabled); + } + + /// Without a metrics exporter there is nowhere to send scrapes, which is reported as + /// `NoListeners` rather than as a disabled receiver. + #[test] + fn test_activation_without_a_metrics_exporter() { + let activation = TelemetryActivation::from_config( + &node_metrics_only(), + &ExporterConfigs::default(), + false, + ); + + assert_eq!(activation.metrics, TelemetryState::NoListeners); + } +} diff --git a/src/init/agent.rs b/src/init/agent.rs index 7a575400..26f99031 100644 --- a/src/init/agent.rs +++ b/src/init/agent.rs @@ -31,6 +31,8 @@ use crate::receivers::kafka::offset_ack_committer::KafkaOffsetCommitter; use crate::receivers::kafka::receiver::KafkaReceiver; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] use crate::receivers::kmsg::receiver::KmsgReceiver; +#[cfg(feature = "node_metrics_receiver")] +use crate::receivers::node_metrics::NodeMetricsReceiver; use crate::receivers::otlp::otlp_grpc::OTLPGrpcServer; use crate::receivers::otlp::otlp_http::OTLPHttpServer; use crate::receivers::otlp_output::OTLPOutput; @@ -1082,6 +1084,17 @@ impl Agent { } }); } + #[cfg(feature = "node_metrics_receiver")] + ReceiverConfig::NodeMetrics(config) => { + let node_metrics_receiver = + NodeMetricsReceiver::new(config.clone(), metrics_output.clone())?; + + // Spawned directly into the shared task set rather than into its own + // supervised set: this receiver owns a single task, so there is + // nothing to supervise or drain. It runs until cancelled, so an early + // completion here legitimately means something went wrong. + node_metrics_receiver.start(&mut receivers_task_set, &receivers_cancel)?; + } } } diff --git a/src/init/args.rs b/src/init/args.rs index 6a00d8c1..e182cd41 100644 --- a/src/init/args.rs +++ b/src/init/args.rs @@ -13,6 +13,8 @@ use crate::init::kafka_exporter::KafkaExporterArgs; use crate::init::kafka_receiver::KafkaReceiverArgs; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] use crate::init::kmsg_receiver::KmsgReceiverArgs; +#[cfg(feature = "node_metrics_receiver")] +use crate::init::node_metrics_receiver::NodeMetricsReceiverArgs; use crate::init::otlp_exporter::OTLPExporterArgs; use crate::init::otlp_receiver::OTLPReceiverArgs; #[cfg(feature = "prometheus")] @@ -82,6 +84,10 @@ pub struct AgentRun { #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] pub kmsg_receiver: KmsgReceiverArgs, + #[command(flatten)] + #[cfg(feature = "node_metrics_receiver")] + pub node_metrics_receiver: NodeMetricsReceiverArgs, + /// Single receiver (type) #[arg(value_enum, long, env = "ROTEL_RECEIVER")] pub receiver: Option, @@ -214,6 +220,8 @@ impl Default for AgentRun { file_receiver: FileReceiverArgs::default(), #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] kmsg_receiver: KmsgReceiverArgs::default(), + #[cfg(feature = "node_metrics_receiver")] + node_metrics_receiver: NodeMetricsReceiverArgs::default(), otlp_with_trace_processor: Vec::new(), otlp_with_logs_processor: Vec::new(), otlp_with_metrics_processor: Vec::new(), @@ -317,6 +325,9 @@ pub enum Receiver { File, #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] Kmsg, + #[cfg(feature = "node_metrics_receiver")] + #[clap(name = "node_metrics")] + NodeMetrics, } impl FromStr for Receiver { @@ -332,6 +343,8 @@ impl FromStr for Receiver { "file" => Ok(Receiver::File), #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] "kmsg" => Ok(Receiver::Kmsg), + #[cfg(feature = "node_metrics_receiver")] + "node_metrics" => Ok(Receiver::NodeMetrics), _ => Err("Unknown receiver"), } } diff --git a/src/init/config.rs b/src/init/config.rs index 11fff59e..9d0fa30d 100644 --- a/src/init/config.rs +++ b/src/init/config.rs @@ -28,6 +28,8 @@ use crate::receivers::fluent::config::FluentReceiverConfig; use crate::receivers::kafka::config::KafkaReceiverConfig; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] use crate::receivers::kmsg::config::KmsgReceiverConfig; +#[cfg(feature = "node_metrics_receiver")] +use crate::receivers::node_metrics::config::NodeMetricsConfig; use crate::receivers::otlp::OTLPReceiverConfig; use figment::{Figment, providers::Env}; use gethostname::gethostname; @@ -214,6 +216,8 @@ pub(crate) enum ReceiverConfig { File(FileReceiverConfig), #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] Kmsg(KmsgReceiverConfig), + #[cfg(feature = "node_metrics_receiver")] + NodeMetrics(NodeMetricsConfig), } impl TryIntoConfig for ExporterArgs { @@ -650,6 +654,10 @@ fn get_receiver_config(config: &AgentRun, receiver: Receiver) -> Result ReceiverConfig::File(config.file_receiver.build_config()), #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] Receiver::Kmsg => ReceiverConfig::Kmsg(config.kmsg_receiver.build_config()), + #[cfg(feature = "node_metrics_receiver")] + Receiver::NodeMetrics => { + ReceiverConfig::NodeMetrics(config.node_metrics_receiver.build_config()?) + } }) } diff --git a/src/init/mod.rs b/src/init/mod.rs index 646d4534..fd1935ac 100644 --- a/src/init/mod.rs +++ b/src/init/mod.rs @@ -17,6 +17,9 @@ mod file_receiver; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] mod kmsg_receiver; +#[cfg(feature = "node_metrics_receiver")] +mod node_metrics_receiver; + mod awsemf_exporter; mod clickhouse_exporter; mod datadog_exporter; diff --git a/src/init/node_metrics_receiver.rs b/src/init/node_metrics_receiver.rs new file mode 100644 index 00000000..91b7342b --- /dev/null +++ b/src/init/node_metrics_receiver.rs @@ -0,0 +1,875 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! CLI integration for the node metrics receiver + +use crate::receivers::node_metrics::collector::matches_path_prefix; +use crate::receivers::node_metrics::config::{ + Collector, DEFAULT_PROCFS_PATH, DEFAULT_SYSFS_PATH, NodeMetricsConfig, +}; +use clap::Args; +use serde::Deserialize; +use std::time::Duration; +use tower::BoxError; + +#[derive(Debug, Args, Clone, Deserialize)] +#[serde(default)] +pub struct NodeMetricsReceiverArgs { + /// Interval between scrapes, as a time duration (minimum 1s) + #[arg( + id("NODE_METRICS_RECEIVER_SCRAPE_INTERVAL"), + long("node-metrics-receiver-scrape-interval"), + env = "ROTEL_NODE_METRICS_RECEIVER_SCRAPE_INTERVAL", + default_value = "60s", + value_parser = humantime::parse_duration + )] + #[serde(with = "humantime_serde")] + pub scrape_interval: Duration, + + /// Enable CPU metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_CPU"), + long("node-metrics-receiver-cpu"), + env = "ROTEL_NODE_METRICS_RECEIVER_CPU", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub cpu: bool, + + /// Enable load average metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_LOADAVG"), + long("node-metrics-receiver-loadavg"), + env = "ROTEL_NODE_METRICS_RECEIVER_LOADAVG", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub loadavg: bool, + + /// Enable memory metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_MEMORY"), + long("node-metrics-receiver-memory"), + env = "ROTEL_NODE_METRICS_RECEIVER_MEMORY", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub memory: bool, + + /// Enable network metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_NETWORK"), + long("node-metrics-receiver-network"), + env = "ROTEL_NODE_METRICS_RECEIVER_NETWORK", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub network: bool, + + /// Enable filesystem metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_FILESYSTEM"), + long("node-metrics-receiver-filesystem"), + env = "ROTEL_NODE_METRICS_RECEIVER_FILESYSTEM", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub filesystem: bool, + + /// Enable system/uname info metrics collection (kernel info, hostname) + #[arg( + id("NODE_METRICS_RECEIVER_UNAME"), + long("node-metrics-receiver-uname"), + env = "ROTEL_NODE_METRICS_RECEIVER_UNAME", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub uname: bool, + + /// Enable kernel stat counters (forks, context switches, interrupts, procs running/blocked from /proc/stat) + #[arg( + id("NODE_METRICS_RECEIVER_STAT"), + long("node-metrics-receiver-stat"), + env = "ROTEL_NODE_METRICS_RECEIVER_STAT", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub stat: bool, + + /// Enable process metrics collection (kernel limits: threads-max, pid_max) + #[arg( + id("NODE_METRICS_RECEIVER_PROCESSES"), + long("node-metrics-receiver-processes"), + env = "ROTEL_NODE_METRICS_RECEIVER_PROCESSES", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub processes: bool, + + /// Enable disk I/O statistics collection (Linux only, reads /proc/diskstats) + #[arg( + id("NODE_METRICS_RECEIVER_DISKSTATS"), + long("node-metrics-receiver-diskstats"), + env = "ROTEL_NODE_METRICS_RECEIVER_DISKSTATS", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub diskstats: bool, + + /// Enable virtual memory statistics collection (/proc/vmstat) + #[arg( + id("NODE_METRICS_RECEIVER_VMSTAT"), + long("node-metrics-receiver-vmstat"), + env = "ROTEL_NODE_METRICS_RECEIVER_VMSTAT", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub vmstat: bool, + + /// Enable network statistics collection (/proc/net/netstat and /proc/net/snmp) + #[arg( + id("NODE_METRICS_RECEIVER_NETSTAT"), + long("node-metrics-receiver-netstat"), + env = "ROTEL_NODE_METRICS_RECEIVER_NETSTAT", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub netstat: bool, + + /// Enable socket statistics collection (/proc/net/sockstat and /proc/net/sockstat6) + #[arg( + id("NODE_METRICS_RECEIVER_SOCKSTAT"), + long("node-metrics-receiver-sockstat"), + env = "ROTEL_NODE_METRICS_RECEIVER_SOCKSTAT", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub sockstat: bool, + + /// Enable file descriptor statistics collection (/proc/sys/fs/file-nr) + #[arg( + id("NODE_METRICS_RECEIVER_FILEFD"), + long("node-metrics-receiver-filefd"), + env = "ROTEL_NODE_METRICS_RECEIVER_FILEFD", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub filefd: bool, + + /// Enable CPU frequency metrics collection (/sys/devices/system/cpu/cpu*/cpufreq/) + #[arg( + id("NODE_METRICS_RECEIVER_CPUFREQ"), + long("node-metrics-receiver-cpufreq"), + env = "ROTEL_NODE_METRICS_RECEIVER_CPUFREQ", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub cpufreq: bool, + + /// Enable thermal zone and cooling device metrics (/sys/class/thermal/) + #[arg( + id("NODE_METRICS_RECEIVER_THERMAL_ZONE"), + long("node-metrics-receiver-thermal-zone"), + env = "ROTEL_NODE_METRICS_RECEIVER_THERMAL_ZONE", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub thermal_zone: bool, + + /// Enable NVMe device info metrics (/sys/class/nvme/) + #[arg( + id("NODE_METRICS_RECEIVER_NVME"), + long("node-metrics-receiver-nvme"), + env = "ROTEL_NODE_METRICS_RECEIVER_NVME", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub nvme: bool, + + /// Enable hardware monitoring sensor metrics (/sys/class/hwmon/) + #[arg( + id("NODE_METRICS_RECEIVER_HWMON"), + long("node-metrics-receiver-hwmon"), + env = "ROTEL_NODE_METRICS_RECEIVER_HWMON", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub hwmon: bool, + + /// Enable time metrics (node_time_seconds, node_boot_time_seconds) + #[arg( + id("NODE_METRICS_RECEIVER_TIME"), + long("node-metrics-receiver-time"), + env = "ROTEL_NODE_METRICS_RECEIVER_TIME", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub time: bool, + + /// Enable textfile collector for custom Prometheus-format metrics + #[arg( + id("NODE_METRICS_RECEIVER_TEXTFILE"), + long("node-metrics-receiver-textfile"), + env = "ROTEL_NODE_METRICS_RECEIVER_TEXTFILE", + action = clap::ArgAction::Set, + default_value = "false" + )] + pub textfile: bool, + + /// Directory or file path for Prometheus-format textfiles (*.prom) + #[arg( + id("NODE_METRICS_RECEIVER_TEXTFILE_DIRECTORY"), + long("node-metrics-receiver-textfile-directory"), + env = "ROTEL_NODE_METRICS_RECEIVER_TEXTFILE_DIRECTORY" + )] + pub textfile_directory: Option, + + /// Root filesystem path prefix, auto-composed onto procfs/sysfs paths + /// (e.g. --node-metrics-receiver-rootfs-path=/host sets procfs to /host/proc and sysfs to /host/sys) + #[arg( + id("NODE_METRICS_RECEIVER_ROOTFS_PATH"), + long("node-metrics-receiver-rootfs-path"), + env = "ROTEL_NODE_METRICS_RECEIVER_ROOTFS_PATH", + default_value = "/" + )] + pub rootfs_path: String, + + /// Path to procfs mount point (for containerized monitoring) + #[arg( + id("NODE_METRICS_RECEIVER_PROCFS_PATH"), + long("node-metrics-receiver-procfs-path"), + env = "ROTEL_NODE_METRICS_RECEIVER_PROCFS_PATH", + default_value = DEFAULT_PROCFS_PATH + )] + pub procfs_path: String, + + /// Path to sysfs mount point (for containerized monitoring) + #[arg( + id("NODE_METRICS_RECEIVER_SYSFS_PATH"), + long("node-metrics-receiver-sysfs-path"), + env = "ROTEL_NODE_METRICS_RECEIVER_SYSFS_PATH", + default_value = DEFAULT_SYSFS_PATH + )] + pub sysfs_path: String, + + /// Service name for the OTLP Resource attribute + #[arg( + id("NODE_METRICS_RECEIVER_SERVICE_NAME"), + long("node-metrics-receiver-service-name"), + env = "ROTEL_NODE_METRICS_RECEIVER_SERVICE_NAME", + default_value = "node_metrics" + )] + pub service_name: String, + + /// Regex pattern to exclude filesystem mount points, in addition to the built-in exclusions + #[arg( + id("NODE_METRICS_RECEIVER_FILESYSTEM_MOUNT_EXCLUDE"), + long("node-metrics-receiver-filesystem-mount-exclude"), + env = "ROTEL_NODE_METRICS_RECEIVER_FILESYSTEM_MOUNT_EXCLUDE" + )] + pub filesystem_mount_exclude: Option, + + /// Regex pattern to include only matching metric names + #[arg( + id("NODE_METRICS_RECEIVER_INCLUDE_FILTER"), + long("node-metrics-receiver-include-filter"), + env = "ROTEL_NODE_METRICS_RECEIVER_INCLUDE_FILTER" + )] + pub include_filter: Option, + + /// Regex pattern to exclude matching metric names + #[arg( + id("NODE_METRICS_RECEIVER_EXCLUDE_FILTER"), + long("node-metrics-receiver-exclude-filter"), + env = "ROTEL_NODE_METRICS_RECEIVER_EXCLUDE_FILTER" + )] + pub exclude_filter: Option, +} + +impl Default for NodeMetricsReceiverArgs { + fn default() -> Self { + Self { + scrape_interval: Duration::from_secs(60), + cpu: true, + loadavg: true, + memory: true, + network: true, + filesystem: true, + uname: true, + stat: true, + processes: true, + diskstats: true, + vmstat: true, + netstat: true, + sockstat: true, + filefd: true, + cpufreq: true, + thermal_zone: true, + nvme: true, + hwmon: true, + time: true, + textfile: false, + textfile_directory: None, + rootfs_path: "/".to_string(), + procfs_path: DEFAULT_PROCFS_PATH.to_string(), + sysfs_path: DEFAULT_SYSFS_PATH.to_string(), + service_name: "node_metrics".to_string(), + filesystem_mount_exclude: None, + include_filter: None, + exclude_filter: None, + } + } +} + +impl NodeMetricsReceiverArgs { + /// Build a NodeMetricsConfig from the CLI args + pub fn build_config(&self) -> Result { + // Collect enabled collectors + let mut collectors = Vec::new(); + if self.cpu { + collectors.push(Collector::Cpu); + } + if self.loadavg { + collectors.push(Collector::Loadavg); + } + if self.memory { + collectors.push(Collector::Memory); + } + if self.network { + collectors.push(Collector::Network); + } + if self.filesystem { + collectors.push(Collector::Filesystem); + } + if self.uname { + collectors.push(Collector::Uname); + } + if self.stat { + collectors.push(Collector::Stat); + } + if self.processes { + collectors.push(Collector::Processes); + } + if self.diskstats { + collectors.push(Collector::Diskstats); + } + if self.vmstat { + collectors.push(Collector::Vmstat); + } + if self.netstat { + collectors.push(Collector::Netstat); + } + if self.sockstat { + collectors.push(Collector::Sockstat); + } + if self.filefd { + collectors.push(Collector::Filefd); + } + if self.cpufreq { + collectors.push(Collector::Cpufreq); + } + if self.thermal_zone { + collectors.push(Collector::ThermalZone); + } + if self.nvme { + collectors.push(Collector::Nvme); + } + if self.hwmon { + collectors.push(Collector::Hwmon); + } + if self.time { + collectors.push(Collector::Time); + } + if self.textfile { + if self.textfile_directory.is_none() { + return Err( + "Textfile collector requires --node-metrics-receiver-textfile-directory".into(), + ); + } + collectors.push(Collector::Textfile); + } else if self.textfile_directory.is_some() { + tracing::warn!( + "--node-metrics-receiver-textfile-directory is set but textfile collector is disabled; \ + add --node-metrics-receiver-textfile true to enable it" + ); + } + + // When rootfs is set to something other than "/", compose it onto the + // procfs and sysfs paths. This allows containerized setups to set a single + // --node-metrics-receiver-rootfs-path=/host instead of separate + // --node-metrics-receiver-procfs-path=/host/proc and + // --node-metrics-receiver-sysfs-path=/host/sys. + // `trim_end_matches` already reduces "/" to "", so only the empty case remains. + let rootfs = self.rootfs_path.trim_end_matches('/'); + + // Reject the combination rather than silently producing "/host/host/proc", which + // would leave every collector reading a path that does not exist. + // Compare with path-component awareness: a plain `starts_with` would reject + // `--procfs-path /hostile/proc` against `--rootfs-path /host`, which is not + // prefixed at all and composes correctly. + if !rootfs.is_empty() + && (matches_path_prefix(&self.procfs_path, rootfs) + || matches_path_prefix(&self.sysfs_path, rootfs)) + { + return Err(format!( + "--node-metrics-receiver-rootfs-path ({}) is prefixed onto the procfs and sysfs \ + paths, so it cannot be combined with an explicit \ + --node-metrics-receiver-procfs-path ({}) or --node-metrics-receiver-sysfs-path \ + ({}). Set either the rootfs path or the individual paths.", + self.rootfs_path, self.procfs_path, self.sysfs_path + ) + .into()); + } + + let compose = |path: &str| -> String { + if rootfs.is_empty() { + path.to_string() + } else { + // Join on a single separator regardless of how the suffix is written + format!("{}/{}", rootfs, path.trim_start_matches('/')) + } + }; + + let procfs_path = compose(&self.procfs_path); + let sysfs_path = compose(&self.sysfs_path); + // Deliberately not composed: the textfile directory is rotel's own data, not a + // kernel pseudo-filesystem, and prefixing it would point at a path that usually + // does not exist inside the host mount. + let textfile_directory = self.textfile_directory.clone(); + + let mut config = NodeMetricsConfig { + scrape_interval: self.scrape_interval, + collectors, + metric_include_filter: None, + metric_exclude_filter: None, + filesystem_mount_exclude: None, + procfs_path, + sysfs_path, + rootfs_path: self.rootfs_path.clone(), + textfile_directory, + service_name: self.service_name.clone(), + }; + + // Set filters if provided + if let Some(ref pattern) = self.include_filter { + config = config.with_include_filter(pattern).map_err(|e| { + format!( + "Invalid --node-metrics-receiver-include-filter regex: {}", + e + ) + })?; + } + + if let Some(ref pattern) = self.exclude_filter { + config = config.with_exclude_filter(pattern).map_err(|e| { + format!( + "Invalid --node-metrics-receiver-exclude-filter regex: {}", + e + ) + })?; + } + + if let Some(ref pattern) = self.filesystem_mount_exclude { + config = config.with_filesystem_mount_exclude(pattern).map_err(|e| { + format!( + "Invalid --node-metrics-receiver-filesystem-mount-exclude regex: {}", + e + ) + })?; + } + + config.normalize_and_validate().map_err(|e| { + format!( + "{} (see --node-metrics-receiver-scrape-interval and the \ + --node-metrics-receiver- flags)", + e + ) + })?; + + Ok(config) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_args() { + let args = NodeMetricsReceiverArgs::default(); + + let config = args.build_config().unwrap(); + assert_eq!(config.scrape_interval, Duration::from_secs(60)); + // Compare the actual collector set: a length check would still pass if one + // collector were swapped for another. + assert_eq!(config.collectors, NodeMetricsConfig::default().collectors); + assert_eq!(config.procfs_path, DEFAULT_PROCFS_PATH); + assert_eq!(config.sysfs_path, DEFAULT_SYSFS_PATH); + } + + /// The hand-written `Default` impl duplicates ~28 clap `default_value` strings. + /// Without this test, changing one and forgetting the other ships a different + /// default than the CLI documents. + #[test] + fn test_clap_defaults_match_default_impl() { + use clap::Parser; + + /// Minimal command that flattens only these args, so clap can be driven with no + /// arguments at all + #[derive(Debug, Parser)] + struct OnlyNodeMetrics { + #[command(flatten)] + args: NodeMetricsReceiverArgs, + } + + let parsed = OnlyNodeMetrics::parse_from(["rotel"]).args; + let defaulted = NodeMetricsReceiverArgs::default(); + + let from_clap = parsed.build_config().unwrap(); + let from_default = defaulted.build_config().unwrap(); + + assert_eq!(from_clap.scrape_interval, from_default.scrape_interval); + assert_eq!(from_clap.collectors, from_default.collectors); + assert_eq!(from_clap.procfs_path, from_default.procfs_path); + assert_eq!(from_clap.sysfs_path, from_default.sysfs_path); + assert_eq!(from_clap.service_name, from_default.service_name); + assert_eq!( + from_clap.textfile_directory, + from_default.textfile_directory + ); + } + + /// The collector toggles must accept an explicit value; clap would otherwise infer + /// `SetTrue` from the `bool` type and make `--...-cpu false` a parse error. + #[test] + fn test_collector_toggles_accept_explicit_values() { + use clap::Parser; + + #[derive(Debug, Parser)] + struct OnlyNodeMetrics { + #[command(flatten)] + args: NodeMetricsReceiverArgs, + } + + let args = OnlyNodeMetrics::parse_from([ + "rotel", + "--node-metrics-receiver-cpu", + "false", + "--node-metrics-receiver-vmstat=false", + ]) + .args; + + assert!(!args.cpu); + assert!(!args.vmstat); + assert!(args.loadavg, "untouched toggles stay enabled"); + + let config = args.build_config().unwrap(); + assert!(!config.collectors.contains(&Collector::Cpu)); + assert!(!config.collectors.contains(&Collector::Vmstat)); + assert!(config.collectors.contains(&Collector::Loadavg)); + } + + #[test] + fn test_rootfs_path_conflicts_with_explicit_procfs_path() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + procfs_path: "/host/proc".to_string(), + ..Default::default() + }; + + // Silently composing these would yield /host/host/proc + let err = args.build_config().unwrap_err().to_string(); + assert!( + err.contains("--node-metrics-receiver-rootfs-path"), + "error should name the conflicting flags, got: {}", + err + ); + } + + #[test] + fn test_rootfs_path_composes_onto_defaults() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host/".to_string(), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert_eq!(config.procfs_path, "/host/proc"); + assert_eq!(config.sysfs_path, "/host/sys"); + } + + /// clap's duplicate-id and conflicting-flag assertions only fire when the whole + /// command is built, which no other test does. This makes the explicit `id(...)` + /// values self-enforcing against a future collision with another receiver's args. + #[test] + fn test_full_cli_definition_is_valid() { + use clap::{CommandFactory, Parser}; + + // Flattening every receiver's and exporter's args is what makes a duplicate id + // or flag collide, so build the whole agent command here. + #[derive(Debug, Parser)] + struct FullAgent { + #[command(flatten)] + agent: crate::init::args::AgentRun, + } + + FullAgent::command().debug_assert(); + } + + /// The toggles take a value, so a bare flag must be rejected rather than silently + /// meaning "true". + #[test] + fn test_bare_collector_toggle_is_an_error() { + use clap::Parser; + + #[derive(Debug, Parser)] + struct OnlyNodeMetrics { + #[command(flatten)] + args: NodeMetricsReceiverArgs, + } + + let result = OnlyNodeMetrics::try_parse_from(["rotel", "--node-metrics-receiver-cpu"]); + assert!( + result.is_err(), + "a bare collector toggle must require a value" + ); + } + + #[test] + fn test_scrape_interval_accepts_duration_strings() { + use clap::Parser; + + #[derive(Debug, Parser)] + struct OnlyNodeMetrics { + #[command(flatten)] + args: NodeMetricsReceiverArgs, + } + + let args = + OnlyNodeMetrics::parse_from(["rotel", "--node-metrics-receiver-scrape-interval", "2m"]) + .args; + assert_eq!(args.scrape_interval, Duration::from_secs(120)); + } + + #[test] + fn test_rootfs_path_conflicts_with_explicit_sysfs_path() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + sysfs_path: "/host/sys".to_string(), + ..Default::default() + }; + + let err = args.build_config().unwrap_err().to_string(); + assert!( + err.contains("--node-metrics-receiver-rootfs-path"), + "error should name the conflicting flags, got: {}", + err + ); + } + + /// A path whose first component merely *begins* with the rootfs string is not under it, + /// so it must compose rather than be rejected. A plain `starts_with` got this wrong. + #[test] + fn test_rootfs_path_accepts_a_path_that_only_shares_a_prefix_string() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + procfs_path: "/hostile/proc".to_string(), + ..Default::default() + }; + + let config = args + .build_config() + .expect("/hostile/proc is not under /host and must be accepted"); + assert_eq!(config.procfs_path, "/host/hostile/proc"); + } + + /// An explicit procfs path that is not under the rootfs prefix is a legitimate + /// combination and must still compose. + #[test] + fn test_rootfs_path_composes_onto_non_default_procfs_path() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + procfs_path: "/proc-alt".to_string(), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert_eq!(config.procfs_path, "/host/proc-alt"); + } + + /// The textfile directory is rotel's own data, so the rootfs prefix must not be + /// applied to it. + #[test] + fn test_rootfs_path_does_not_compose_onto_textfile_directory() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + textfile: true, + textfile_directory: Some("/var/lib/rotel/textfile".to_string()), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert_eq!( + config.textfile_directory.as_deref(), + Some("/var/lib/rotel/textfile") + ); + } + + /// The receiver layer reports the property; the flag name is added here. + #[test] + fn test_validation_error_points_at_the_flags() { + let args = NodeMetricsReceiverArgs { + scrape_interval: Duration::from_millis(500), + ..Default::default() + }; + + let err = args.build_config().unwrap_err().to_string(); + assert!( + err.contains("--node-metrics-receiver-"), + "error should point at the receiver's flags, got: {}", + err + ); + assert!( + err.contains("500ms"), + "error should report the value, got: {}", + err + ); + } + + #[test] + fn test_selective_collectors() { + let args = NodeMetricsReceiverArgs { + scrape_interval: Duration::from_secs(30), + cpu: false, + loadavg: true, + memory: true, + network: false, + filesystem: false, + uname: false, + stat: false, + processes: false, + diskstats: false, + vmstat: false, + netstat: false, + sockstat: false, + filefd: false, + cpufreq: false, + thermal_zone: false, + nvme: false, + hwmon: false, + time: false, + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert_eq!( + config.collectors, + vec![Collector::Loadavg, Collector::Memory] + ); + } + + #[test] + fn test_with_filters() { + let args = NodeMetricsReceiverArgs { + loadavg: true, + include_filter: Some("node_load.*".to_string()), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + // Assert the filter's effect, not just that one was compiled + assert!(config.should_include_metric("node_load1")); + assert!(!config.should_include_metric("node_cpu_seconds_total")); + } + + #[test] + fn test_no_collectors_error() { + let args = NodeMetricsReceiverArgs { + cpu: false, + loadavg: false, + memory: false, + network: false, + filesystem: false, + uname: false, + stat: false, + processes: false, + diskstats: false, + vmstat: false, + netstat: false, + sockstat: false, + filefd: false, + cpufreq: false, + thermal_zone: false, + nvme: false, + hwmon: false, + time: false, + ..Default::default() + }; + + let result = args.build_config(); + assert!(result.is_err()); + } + + #[test] + fn test_invalid_filter_regex() { + let args = NodeMetricsReceiverArgs { + include_filter: Some("[invalid".to_string()), + ..Default::default() + }; + + // The whole point of the wrapper is to name the flag + let err = args.build_config().unwrap_err().to_string(); + assert!( + err.contains("--node-metrics-receiver-include-filter"), + "error should name the offending flag, got: {}", + err + ); + } + + #[test] + fn test_filesystem_mount_exclude() { + let args = NodeMetricsReceiverArgs { + filesystem_mount_exclude: Some("^/nfs/".to_string()), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert!(config.filesystem_mount_exclude.is_some()); + let re = config.filesystem_mount_exclude.as_ref().unwrap(); + assert!(re.is_match("/nfs/share")); + assert!(!re.is_match("/mnt/data")); + } + + #[test] + fn test_textfile_requires_directory() { + let args = NodeMetricsReceiverArgs { + textfile: true, + textfile_directory: None, + ..Default::default() + }; + + let result = args.build_config(); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("textfile-directory") + ); + } + + #[test] + fn test_textfile_with_directory() { + let args = NodeMetricsReceiverArgs { + textfile: true, + textfile_directory: Some("/var/lib/node_exporter/textfile".to_string()), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert!(config.collectors.contains(&Collector::Textfile)); + assert_eq!( + config.textfile_directory, + Some("/var/lib/node_exporter/textfile".to_string()) + ); + } +} diff --git a/src/receivers/mod.rs b/src/receivers/mod.rs index 56be3db0..7b3287f5 100644 --- a/src/receivers/mod.rs +++ b/src/receivers/mod.rs @@ -14,6 +14,9 @@ pub mod file; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] pub mod kmsg; +#[cfg(feature = "node_metrics_receiver")] +pub mod node_metrics; + use opentelemetry::global; use opentelemetry::metrics::Meter; diff --git a/src/receivers/node_metrics/collector/mod.rs b/src/receivers/node_metrics/collector/mod.rs new file mode 100644 index 00000000..5bcbce99 --- /dev/null +++ b/src/receivers/node_metrics/collector/mod.rs @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! System metrics collector reading directly from /proc and /sys +//! +//! This collector reads Linux kernel interfaces, emitting raw counter values +//! compatible with Prometheus node_exporter conventions. + +// Notice: portions of this collector are derived from Prometheus node_exporter +// (https://github.com/prometheus/node_exporter), licensed under the Apache License 2.0. +/* Copyright © The Prometheus Authors */ +// +// Derived material, by submodule: +// procfs.rs metric names and HELP strings; the default filesystem-type and +// mount-point exclusion lists +// sysfs.rs metric names and HELP strings; the hwmon chip-naming approach from +// that project's `collector/hwmon.go` +// textfile.rs the textfile collector's behaviour and its `node_textfile_*` semantics +// util.rs the virtual-device and partition exclusion heuristics +// +// Where behaviour deliberately differs from node_exporter, the relevant function says so. + +mod procfs; +mod sysfs; +mod textfile; +mod util; + +pub(crate) use util::matches_path_prefix; + +#[cfg(test)] +mod test_support; + +use crate::receivers::node_metrics::config::{Collector, NodeMetricsConfig}; +use std::fs; +use std::time::{SystemTime, UNIX_EPOCH}; +// Use portable-atomic on platforms without native 64-bit atomics (e.g. 32-bit ARM) +#[cfg(not(target_has_atomic = "64"))] +use portable_atomic::{AtomicU64, Ordering}; +#[cfg(target_has_atomic = "64")] +use std::sync::atomic::{AtomicU64, Ordering}; +use tracing::{debug, info, warn}; + +/// Collected metric value with optional labels +/// +/// Metric names, units and descriptions are allocated afresh on every scrape. +#[derive(Debug, Clone)] +pub struct CollectedMetric { + pub name: String, + pub value: f64, + pub labels: Vec<(String, String)>, + pub metric_type: MetricType, + pub unit: Option, + pub description: Option, +} + +/// Type of metric (determines how it's represented in OTLP) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum MetricType { + /// Gauge - current value that can go up or down + Gauge, + /// Counter - monotonically increasing value + Counter, +} + +/// System metrics collector reading from /proc and /sys +pub struct SystemCollector { + /// Clock ticks per second (for CPU time conversion) + clk_tck: f64, + /// Memory page size in bytes (for sockstat mem-to-bytes conversion) + page_size: f64, + /// System boot time in seconds since epoch, or 0 while it is unknown. + /// + /// Re-read on every scrape that reads `/proc/stat`, because the kernel's `btime` + /// moves whenever the wall clock is stepped — a device with no RTC boots near the + /// epoch and then jumps when NTP syncs, and a value latched before that would be + /// wrong forever. + boot_time: AtomicU64, + /// Boot time used as `start_time_unix_nano` for cumulative counters, or 0 while + /// unknown. + /// + /// Unlike `boot_time` this is latched at the first non-zero reading: the counters + /// themselves do not reset when the clock is stepped, and moving their start time + /// would be reported downstream as a counter reset. + /// + /// One consequence: if the clock is later stepped *backwards* past this value, counters + /// briefly carry a start time ahead of their own timestamp, which a strict backend may + /// reject. It resolves itself on the next forward correction, and the alternative — + /// re-latching — would signal a reset on every clock adjustment. + counter_start_time: AtomicU64, + /// Path to procfs mount point + procfs_path: String, + /// Path to sysfs mount point + sysfs_path: String, +} + +impl SystemCollector { + /// Create a new system collector with configurable procfs and sysfs paths + pub fn new(procfs_path: &str, sysfs_path: &str) -> Self { + // Get clock ticks per second for CPU time conversion. + // sysconf returns -1 on error; fall back to the POSIX-common default of 100. + let raw_clk = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; + let clk_tck = if raw_clk > 0 { raw_clk as f64 } else { 100.0 }; + // Get memory page size for sockstat mem-to-bytes conversion. + // sysconf returns -1 on error; fall back to the common default of 4096. + let raw_page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + let page_size = if raw_page > 0 { + raw_page as f64 + } else { + 4096.0 + }; + + // Get boot time from /proc/stat + let boot_time = Self::read_boot_time(procfs_path).unwrap_or_else(|| { + warn!( + "Could not read boot time from {}/stat; \ + counter start_time will default to epoch (0) until it can be read", + procfs_path + ); + 0 + }); + + Self { + clk_tck, + page_size, + boot_time: AtomicU64::new(boot_time), + counter_start_time: AtomicU64::new(boot_time), + procfs_path: procfs_path.to_string(), + sysfs_path: sysfs_path.to_string(), + } + } + + /// Read boot time from /proc/stat + fn read_boot_time(procfs_path: &str) -> Option { + let path = format!("{}/stat", procfs_path); + let content = fs::read_to_string(path).ok()?; + Self::parse_boot_time(&content) + } + + /// Extract the `btime` field from the contents of /proc/stat + fn parse_boot_time(stat_content: &str) -> Option { + for line in stat_content.lines() { + if let Some(rest) = line.strip_prefix("btime ") { + // The kernel formats btime from a *signed* timespec64 as %llu, so a machine + // whose wall clock is near the epoch prints a wrapped value (~1.8e19) after + // a backwards clock step or a suspend/resume. Accepting it would publish a + // nonsense boot time and, worse, latch it as every counter's start time. + let btime: i64 = rest.trim().parse().ok()?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(i64::MAX); + if btime <= 0 || btime > now { + warn!( + "Ignoring implausible boot time from /proc/stat: btime {}", + rest.trim() + ); + return None; + } + return Some(btime as u64); + } + } + None + } + + /// Get the current boot time in seconds since epoch, or 0 if it is not known + pub fn boot_time(&self) -> u64 { + self.boot_time.load(Ordering::Relaxed) + } + + /// Get the boot time to use as `start_time_unix_nano` for cumulative counters, or 0 + /// if it is not known. + /// + /// Latched at the first known value so that a wall-clock step does not look like a + /// counter reset downstream. + pub fn counter_start_time(&self) -> u64 { + self.counter_start_time.load(Ordering::Relaxed) + } + + /// Collect all enabled metrics based on config + pub fn collect(&self, config: &NodeMetricsConfig) -> Vec { + // A default collector set on a many-core host produces well over a thousand + // metrics (vmstat and netstat alone contribute several hundred). + let mut metrics = Vec::with_capacity(2048); + + // Read /proc/stat once for the collectors that need it and for the boot time. + // `Time` is included because `node_boot_time_seconds` comes from the `btime` line: + // without it, a clock step would go unnoticed whenever cpu and stat are disabled. + let stat_content = if self.counter_start_time() == 0 + || config + .collectors + .iter() + .any(|c| matches!(c, Collector::Cpu | Collector::Stat | Collector::Time)) + { + let path = format!("{}/stat", self.procfs_path); + match fs::read_to_string(&path) { + Ok(c) => Some(c), + Err(e) => { + debug!("Failed to read {}: {}", path, e); + None + } + } + } else { + None + }; + + // Refresh the boot time every scrape: the kernel's `btime` moves whenever the wall + // clock is stepped, and procfs may only have become readable after startup. + if let Some(btime) = stat_content.as_deref().and_then(Self::parse_boot_time) { + let previous = self.boot_time.swap(btime, Ordering::Relaxed); + // btime tracks the timekeeper's wall-clock offset, so it also drifts by a + // second under ordinary slewing, suspend/resume or VM migration. Only a real + // step is worth an info line; the rest is debug. + if previous.abs_diff(btime) > 1 { + info!("System boot time is {} (previously {})", btime, previous); + } else if previous != btime { + debug!("System boot time adjusted to {} (was {})", btime, previous); + } + // Latch the counter start time at the first known value, so a clock step is + // not reported downstream as a counter reset. + let _ = self.counter_start_time.compare_exchange( + 0, + btime, + Ordering::Relaxed, + Ordering::Relaxed, + ); + } + + for collector in &config.collectors { + let result = match collector { + Collector::Cpu => self.collect_cpu(config, stat_content.as_deref()), + Collector::Loadavg => self.collect_loadavg(config), + Collector::Memory => self.collect_meminfo(config), + Collector::Network => self.collect_netdev(config), + Collector::Filesystem => self.collect_filesystem(config), + Collector::Uname => self.collect_uname(config), + Collector::Stat => self.collect_stat(config, stat_content.as_deref()), + Collector::Processes => self.collect_processes(config), + Collector::Diskstats => self.collect_diskstats(config), + Collector::Vmstat => self.collect_vmstat(config), + Collector::Netstat => self.collect_netstat(config), + Collector::Sockstat => self.collect_sockstat(config), + Collector::Filefd => self.collect_filefd(config), + Collector::Cpufreq => self.collect_cpufreq(config), + Collector::ThermalZone => self.collect_thermal_zone(config), + Collector::Nvme => self.collect_nvme(config), + Collector::Hwmon => self.collect_hwmon(config), + Collector::Textfile => self.collect_textfile(config), + Collector::Time => self.collect_time(config), + }; + metrics.extend(result); + } + + metrics + } +} + +impl Default for SystemCollector { + fn default() -> Self { + Self::new("/proc", "/sys") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::receivers::node_metrics::collector::test_support::*; + + // --------------------------------------------------------------------- + // Construction + // --------------------------------------------------------------------- + + #[test] + fn test_boot_time_read_from_stat() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "stat", STAT_FIXTURE); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + assert_eq!(collector.boot_time(), 1_700_000_000); + + // Without a readable /proc/stat the boot time stays unknown rather than + // being invented; collect() retries the read on every scrape. + let empty = tempfile::tempdir().unwrap(); + let without_stat = SystemCollector::new(empty.path().to_str().unwrap(), "/sys"); + assert_eq!(without_stat.boot_time(), 0); + } + + #[test] + fn test_boot_time_retried_on_collect_until_readable() { + // A procfs that is not mounted yet when the receiver starts: without the + // per-scrape retry every counter would keep epoch 0 as its start time for + // the whole life of the process. + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + let collector = SystemCollector::new(&path, &path); + assert_eq!(collector.boot_time(), 0); + + // While boot time is unknown the time collector suppresses the metric + // rather than reporting a plausible-looking 1970 boot. + let before = collector.collect(&config_for(vec![Collector::Time])); + assert!(matching(&before, "node_boot_time_seconds", &[]).is_empty()); + find_one(&before, "node_time_seconds", &[]); + + // procfs appears between two scrapes. + write(tmp.path(), "stat", STAT_FIXTURE); + let after = collector.collect(&config_for(vec![Collector::Time])); + + assert_eq!(collector.boot_time(), 1_700_000_000); + let boot = find_one(&after, "node_boot_time_seconds", &[]); + assert_eq!(boot.value, 1_700_000_000.0); + assert_eq!(boot.unit.as_deref(), Some("s")); + assert_eq!(boot.metric_type, MetricType::Gauge); + assert!(boot.labels.is_empty()); + } + + /// The kernel formats `btime` from a signed value as `%llu`, so a machine whose clock + /// is near the epoch can print a wrapped value. Accepting it would publish a boot time + /// ~580 billion years in the future and latch it into every counter's start time. + #[test] + fn test_implausible_boot_time_is_rejected() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + // 2^64 - 10, i.e. -10 seconds printed as unsigned + write( + tmp.path(), + "stat", + "btime 18446744073709551606\ncpu 1 2 3 4 5 6 7 8 9 10\n", + ); + + let collector = SystemCollector::new(&path, &path); + assert_eq!(collector.boot_time(), 0, "a wrapped btime must be rejected"); + assert_eq!(collector.counter_start_time(), 0); + + // A boot time in the future is equally implausible + write(tmp.path(), "stat", "btime 99999999999\n"); + let metrics = collector.collect(&config_for(vec![Collector::Time])); + assert_eq!(collector.boot_time(), 0); + assert!(matching(&metrics, "node_boot_time_seconds", &[]).is_empty()); + + // ...and a plausible one is still accepted + write(tmp.path(), "stat", STAT_FIXTURE); + collector.collect(&config_for(vec![Collector::Time])); + assert_eq!(collector.boot_time(), 1_700_000_000); + } + + /// A wall-clock step moves the kernel's `btime`, so the reported boot time must track + /// it — while the start time stamped on cumulative counters stays put, since moving it + /// would be read downstream as a counter reset. + #[test] + fn test_boot_time_tracks_clock_steps_but_counter_start_time_is_latched() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + write(tmp.path(), "stat", STAT_FIXTURE); + + let collector = SystemCollector::new(&path, &path); + assert_eq!(collector.boot_time(), 1_700_000_000); + assert_eq!(collector.counter_start_time(), 1_700_000_000); + + // The clock is stepped forward, so the kernel reports a later boot time. This is + // what an RTC-less device does when NTP first syncs. + write( + tmp.path(), + "stat", + &STAT_FIXTURE.replace("btime 1700000000", "btime 1700000500"), + ); + let after = collector.collect(&config_for(vec![Collector::Time])); + + // The gauge follows the kernel... + assert_eq!(collector.boot_time(), 1_700_000_500); + assert_eq!( + find_one(&after, "node_boot_time_seconds", &[]).value, + 1_700_000_500.0 + ); + // ...but the counter start time keeps its first known value. + assert_eq!(collector.counter_start_time(), 1_700_000_000); + } + + /// `/proc/stat` must be read when the `time` collector is enabled, even with cpu and + /// stat disabled and a boot time already latched — otherwise a clock step goes unseen. + #[test] + fn test_time_collector_alone_still_refreshes_boot_time() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + write(tmp.path(), "stat", STAT_FIXTURE); + + let collector = SystemCollector::new(&path, &path); + assert_eq!(collector.counter_start_time(), 1_700_000_000); + + write( + tmp.path(), + "stat", + &STAT_FIXTURE.replace("btime 1700000000", "btime 1700009999"), + ); + // Only Time — neither Cpu nor Stat forces the /proc/stat read here. + collector.collect(&config_for(vec![Collector::Time])); + + assert_eq!(collector.boot_time(), 1_700_009_999); + } + + // --------------------------------------------------------------------- + // Top-level dispatch + // --------------------------------------------------------------------- + + #[test] + fn test_collect_with_filter() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "loadavg", "0.52 0.58 0.59 2/1234 5678\n"); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let config = NodeMetricsConfig::new(60) + .with_include_filter("^node_load1$") + .unwrap(); + + let metrics = collector.collect_loadavg(&config); + + assert_eq!(metrics.len(), 1); + assert_eq!(metrics[0].name, "node_load1"); + assert_eq!(metrics[0].value, 0.52); + } + + #[test] + fn test_collect_dispatches_to_every_enabled_collector() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "stat", STAT_FIXTURE); + + let path = tmp.path().to_str().unwrap(); + let collector = SystemCollector::new(path, path); + let metrics = collector.collect(&config_for(vec![Collector::Cpu, Collector::Stat])); + + // A single collect() call returns the metrics of both collectors, from one + // read of /proc/stat. + assert_eq!( + find_one( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "0"), ("mode", "user")] + ) + .value, + 1000.0 / collector.clk_tck + ); + assert_eq!(find_one(&metrics, "node_forks_total", &[]).value, 4242.0); + } + + #[test] + fn test_collect_without_stat_file() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "loadavg", "1.5 1.0 0.5 1/100 200\n"); + + let path = tmp.path().to_str().unwrap(); + let collector = SystemCollector::new(path, path); + let metrics = collector.collect(&config_for(vec![Collector::Loadavg])); + + assert_eq!(metrics.len(), 3); + assert_eq!(find_one(&metrics, "node_load1", &[]).value, 1.5); + // The boot-time retry finds nothing to read and leaves it unknown. + assert_eq!(collector.boot_time(), 0); + } + + #[test] + fn test_collect_empty_fixture_reports_only_syscall_backed_metrics() { + let procfs = tempfile::tempdir().unwrap(); + let sysfs = tempfile::tempdir().unwrap(); + + let mut collectors = NodeMetricsConfig::default().collectors; + collectors.push(Collector::Textfile); + + let collector = SystemCollector::new( + procfs.path().to_str().unwrap(), + sysfs.path().to_str().unwrap(), + ); + let metrics = collector.collect(&config_for(collectors)); + + // Every file-backed collector finds nothing and reports nothing, without + // panicking. Only uname and the wall clock, which read no files, remain; + // node_boot_time_seconds is suppressed because btime is unknown. + let mut names: Vec<&str> = metrics.iter().map(|m| m.name.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["node_time_seconds", "node_uname_info"]); + } +} diff --git a/src/receivers/node_metrics/collector/procfs.rs b/src/receivers/node_metrics/collector/procfs.rs new file mode 100644 index 00000000..be740985 --- /dev/null +++ b/src/receivers/node_metrics/collector/procfs.rs @@ -0,0 +1,2575 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Collectors reading `/proc`: CPU, load average, memory, network, kernel stat counters, +//! processes, disk I/O, vmstat, netstat and socket/file-descriptor statistics. +//! +//! Three members are wholly or partly syscall-backed but live here because they belong to +//! the same host-level group: `collect_uname` calls `uname(2)`, `collect_time` reads the +//! wall clock, and `collect_filesystem` takes the mount list from `/proc/mounts` but its +//! values from `statfs(2)`. +//! +//! Metric names and HELP strings, and the filesystem exclusion lists, are derived from +//! Prometheus node_exporter — see the notice in this module's `mod.rs`. + +use crate::receivers::node_metrics::collector::util::{ + decode_mount_path, is_partition, is_virtual_disk, matches_path_prefix, utsname_field, +}; +use crate::receivers::node_metrics::collector::{CollectedMetric, MetricType, SystemCollector}; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use std::fs; +use tracing::{debug, warn}; + +// On Linux use the LFS variant: plain `statfs` is the 32-bit interface on glibc and returns +// EOVERFLOW for a filesystem whose block or inode counts do not fit, which would drop large +// filesystems entirely on the 32-bit targets this receiver supports. The BSD `statfs` is +// already 64-bit. +#[cfg(not(target_os = "linux"))] +use libc::{statfs, statfs as StatFs}; +#[cfg(target_os = "linux")] +use libc::{statfs64 as StatFs, statfs64 as statfs}; + +impl SystemCollector { + /// Collect CPU metrics from /proc/stat + /// Emits `node_cpu_seconds_total` (modes: user..steal) and + /// `node_cpu_guest_seconds_total` (modes: user, nice), matching + /// Prometheus node_exporter conventions. + pub(super) fn collect_cpu( + &self, + config: &NodeMetricsConfig, + stat_content: Option<&str>, + ) -> Vec { + let mut metrics = Vec::new(); + + let content = match stat_content { + Some(c) => c, + None => return metrics, + }; + + // /proc/stat fields (indices 1..=10): + // user nice system idle iowait irq softirq steal guest guest_nice + // Fields 1-8 -> node_cpu_seconds_total + // Fields 9-10 -> node_cpu_guest_seconds_total (modes: user, nice) + let cpu_modes = [ + "user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", + ]; + let guest_modes = ["user", "nice"]; + + for line in content.lines() { + // Match cpu lines (cpu, cpu0, cpu1, etc.) + if !line.starts_with("cpu") { + continue; + } + + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.is_empty() { + continue; + } + + let cpu_label = parts[0]; + + // Skip the aggregate "cpu" line — only report per-core metrics, + // matching Prometheus node_exporter behaviour. + if cpu_label == "cpu" { + continue; + } + let cpu_num = if let Some(num) = cpu_label.strip_prefix("cpu") { + num.to_string() + } else { + continue; + }; + + // node_cpu_seconds_total: fields 1-8 + if config.should_include_metric("node_cpu_seconds_total") { + for (i, mode) in cpu_modes.iter().enumerate() { + let field_idx = i + 1; // user=1, nice=2, ... softirq=7, steal=8 + if field_idx >= parts.len() { + break; + } + // Skip rather than substitute 0: for a monotonic counter a fabricated + // zero looks like a counter reset and makes rate() spike. + let Ok(ticks) = parts[field_idx].parse::() else { + continue; + }; + let seconds = ticks / self.clk_tck; + + metrics.push(CollectedMetric { + name: "node_cpu_seconds_total".to_string(), + value: seconds, + labels: vec![ + ("cpu".to_string(), cpu_num.clone()), + ("mode".to_string(), mode.to_string()), + ], + metric_type: MetricType::Counter, + unit: Some("s".to_string()), + description: Some("Seconds the CPUs spent in each mode".to_string()), + }); + } + } + + // node_cpu_guest_seconds_total: fields 9-10 + if config.should_include_metric("node_cpu_guest_seconds_total") { + for (i, mode) in guest_modes.iter().enumerate() { + let field_idx = 9 + i; // guest=9, guest_nice=10 + if field_idx >= parts.len() { + break; + } + // Skip rather than substitute 0: for a monotonic counter a fabricated + // zero looks like a counter reset and makes rate() spike. + let Ok(ticks) = parts[field_idx].parse::() else { + continue; + }; + let seconds = ticks / self.clk_tck; + + metrics.push(CollectedMetric { + name: "node_cpu_guest_seconds_total".to_string(), + value: seconds, + labels: vec![ + ("cpu".to_string(), cpu_num.clone()), + ("mode".to_string(), mode.to_string()), + ], + metric_type: MetricType::Counter, + unit: Some("s".to_string()), + description: Some("Seconds the CPUs spent in guest mode".to_string()), + }); + } + } + } + + metrics + } + + /// Collect load average from /proc/loadavg + pub(super) fn collect_loadavg(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/loadavg", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + let parts: Vec<&str> = content.split_whitespace().collect(); + if parts.len() < 3 { + return metrics; + } + + let loads = [ + ("node_load1", parts[0], "1m load average"), + ("node_load5", parts[1], "5m load average"), + ("node_load15", parts[2], "15m load average"), + ]; + + for (name, value_str, description) in loads { + if config.should_include_metric(name) + && let Ok(value) = value_str.parse::() + { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some(description.to_string()), + }); + } + } + + metrics + } + + /// Collect memory metrics from /proc/meminfo + /// Dynamically reads ALL fields, converting kB to bytes + pub(super) fn collect_meminfo(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/meminfo", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + for line in content.lines() { + // Format: "FieldName: 12345 kB" or "FieldName: 12345" + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 2 { + continue; + } + + // Normalize parenthesized suffixes: "Active(anon)" -> "Active_anon" + let field_name = parts[0] + .trim_end_matches(':') + .replace('(', "_") + .replace(')', ""); + let value: f64 = match parts[1].parse() { + Ok(v) => v, + Err(_) => continue, + }; + + // Check if value is in kB (most fields are) + let (final_value, has_unit) = if parts.len() >= 3 && parts[2] == "kB" { + (value * 1024.0, true) // Convert kB to bytes + } else { + (value, false) + }; + + // Convert field name to metric name: "MemTotal" -> "node_memory_MemTotal_bytes" + let metric_name = if has_unit { + format!("node_memory_{}_bytes", field_name) + } else { + format!("node_memory_{}", field_name) + }; + + if config.should_include_metric(&metric_name) { + metrics.push(CollectedMetric { + name: metric_name, + value: final_value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: if has_unit { + Some("By".to_string()) + } else { + None + }, + description: None, + }); + } + } + + metrics + } + + /// Collect network device metrics from /proc/net/dev + pub(super) fn collect_netdev(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/net/dev", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + // Skip 2 header lines: + // "Inter-| Receive ..." + // " face |bytes packets ..." + for line in content.lines().skip(2) { + // Split on ':' first to cleanly separate interface name from values. + // The kernel format is "%6s: %8lu ..." so the colon is always present. + let (iface, rest) = match line.split_once(':') { + Some(pair) => pair, + None => continue, + }; + let interface = iface.trim(); + + let parts: Vec<&str> = rest.split_whitespace().collect(); + if parts.len() < 16 { + continue; + } + + let labels = vec![("device".to_string(), interface.to_string())]; + + // Receive fields (columns 0-7): bytes, packets, errs, drop, fifo, frame, compressed, multicast + // Transmit fields (columns 8-15): bytes, packets, errs, drop, fifo, colls, carrier, compressed + let netdev_metrics = [ + ( + "node_network_receive_bytes_total", + parts[0], + "By", + "Network device statistic receive_bytes", + ), + ( + "node_network_receive_packets_total", + parts[1], + "{packets}", + "Network device statistic receive_packets", + ), + ( + "node_network_receive_errs_total", + parts[2], + "{errors}", + "Network device statistic receive_errs", + ), + ( + "node_network_receive_drop_total", + parts[3], + "{packets}", + "Network device statistic receive_drop", + ), + ( + "node_network_receive_fifo_total", + parts[4], + "{events}", + "Network device statistic receive_fifo", + ), + ( + "node_network_receive_frame_total", + parts[5], + "{events}", + "Network device statistic receive_frame", + ), + ( + "node_network_receive_compressed_total", + parts[6], + "{packets}", + "Network device statistic receive_compressed", + ), + ( + "node_network_receive_multicast_total", + parts[7], + "{packets}", + "Network device statistic receive_multicast", + ), + ( + "node_network_transmit_bytes_total", + parts[8], + "By", + "Network device statistic transmit_bytes", + ), + ( + "node_network_transmit_packets_total", + parts[9], + "{packets}", + "Network device statistic transmit_packets", + ), + ( + "node_network_transmit_errs_total", + parts[10], + "{errors}", + "Network device statistic transmit_errs", + ), + ( + "node_network_transmit_drop_total", + parts[11], + "{packets}", + "Network device statistic transmit_drop", + ), + ( + "node_network_transmit_fifo_total", + parts[12], + "{events}", + "Network device statistic transmit_fifo", + ), + ( + "node_network_transmit_colls_total", + parts[13], + "{events}", + "Network device statistic transmit_colls", + ), + ( + "node_network_transmit_carrier_total", + parts[14], + "{events}", + "Network device statistic transmit_carrier", + ), + ( + "node_network_transmit_compressed_total", + parts[15], + "{packets}", + "Network device statistic transmit_compressed", + ), + ]; + + for (name, value_str, unit, description) in netdev_metrics { + if config.should_include_metric(name) + && let Ok(value) = value_str.parse::() + { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type: MetricType::Counter, + unit: Some(unit.to_string()), + description: Some(description.to_string()), + }); + } + } + } + + metrics + } + + /// Collect filesystem metrics using statfs + pub(super) fn collect_filesystem(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Prefer PID 1's mount table, as node_exporter does. `{procfs}/mounts` is a symlink + // to `self/mounts`, so with a bind-mounted host procfs it still yields *this* + // process's mount namespace — i.e. the container's — which would report the + // container's bind mounts labelled as if they were the host's filesystems. + let init_path = format!("{}/1/mounts", self.procfs_path); + let self_path = format!("{}/mounts", self.procfs_path); + let (path, content) = match fs::read_to_string(&init_path) { + Ok(c) => (init_path, c), + Err(e) => { + debug!( + "Failed to read {}: {}; falling back to {}", + init_path, e, self_path + ); + match fs::read_to_string(&self_path) { + Ok(c) => (self_path, c), + Err(e) => { + debug!("Failed to read {}: {}", self_path, e); + return metrics; + } + } + } + }; + debug!("Read mount table from {}", path); + + // Filesystem types to ignore (virtual filesystems) + let ignore_fstypes = [ + "sysfs", + "proc", + "devtmpfs", + "devpts", + "securityfs", + "cgroup", + "cgroup2", + "pstore", + "debugfs", + "hugetlbfs", + "mqueue", + "configfs", + "binfmt_misc", + "fusectl", + "fuse.lxcfs", // virtual, checked before the remote list + "overlay", + "nsfs", + "squashfs", + "autofs", + "tracefs", + "bpf", + "rpc_pipefs", + "selinuxfs", + "iso9660", + "procfs", // BSD-style procfs type name + ]; + + // Network filesystem types are skipped by default: `statfs` on a mount whose + // server has gone away blocks in the kernel uninterruptibly, and the scrape + // runs on a shared blocking thread that cannot be aborted. Local mounts can + // still be excluded individually with `filesystem_mount_exclude`. + let ignore_remote_fstypes = [ + "nfs", + "nfs4", + "cifs", + "smb3", + "smbfs", + "glusterfs", + "ceph", + "lustre", + "beegfs", + "gpfs", + "davfs", + "davfs2", + "9p", + "afs", + "ncpfs", + "orangefs", + "coda", + // Shared-disk cluster filesystems rather than network mounts, but excluded for + // the same reason: cluster locking can make statfs block indefinitely. + "ocfs2", + "gfs2", + // FUSE mounts are named by subtype; only the network-backed ones are + // excluded, so local FUSE filesystems (mergerfs, gocryptfs, encfs, bindfs) + // keep reporting. + "fuse.sshfs", + "fuse.s3fs", + "fuse.rclone", + "fuse.glusterfs", + "fuse.cephfs", + "fuse.davfs2", + "fuse.smbnetfs", + "fuse.curlftpfs", + "fuse.juicefs", + "fuse.gcsfuse", + "fuse.blobfuse2", + "fuse.goofys", + "fuse.s3ql", + "fuse.ceph-fuse", + "fuse.mfs", + ]; + + // Mount points to ignore (matched with path-boundary awareness: + // a prefix matches if the mount point equals it exactly or continues + // with '/', preventing "/dev" from matching "/developer"). + // Excluded including the mount point itself: these are kernel pseudo-filesystems. + let ignore_mount_paths = ["/proc", "/sys", "/dev"]; + // Excluded only *below* the path: a filesystem mounted exactly here is a real + // volume worth reporting (a dedicated /var/lib/docker disk is precisely the one + // most likely to fill up). This follows node_exporter for the docker and + // credentials paths, and is deliberately broader for /var/lib/containers (where it + // excludes only .../storage) and /run/user (which it does not exclude at all). + let ignore_mount_subpaths = [ + "/run/credentials", + "/run/user", + "/var/lib/docker", + "/var/lib/containers", + ]; + + // Deduplicate by (device, mountpoint), matching Prometheus node_exporter. + // /proc/mounts can contain duplicate entries from bind mounts or mount propagation. + let mut seen = std::collections::HashSet::new(); + + for line in content.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 4 { + continue; + } + + let device = decode_mount_path(parts[0]); + let mount_point = decode_mount_path(parts[1]); + let fstype = parts[2]; + // The mount options carry the read-only state, so it does not need a second + // syscall — and `statfs.f_flags` does not exist on Linux, only on the BSDs. + let read_only = parts[3].split(',').any(|opt| opt == "ro"); + + if !seen.insert((device.clone(), mount_point.clone())) { + continue; + } + + // Skip ignored filesystem types + if ignore_fstypes.contains(&fstype) { + continue; + } + + // Skip network filesystems, where statfs can block indefinitely + if ignore_remote_fstypes.contains(&fstype) { + debug!( + "Skipping remote filesystem {} of type {}", + mount_point, fstype + ); + continue; + } + + // Skip ignored mount point prefixes (with path boundary check: + // "/dev" matches "/dev" and "/dev/shm" but not "/developer") + if ignore_mount_paths + .iter() + .any(|prefix| matches_path_prefix(&mount_point, prefix)) + { + continue; + } + + if ignore_mount_subpaths + .iter() + .any(|prefix| mount_point != *prefix && matches_path_prefix(&mount_point, prefix)) + { + continue; + } + + // Skip mount points matching the user-provided exclude regex + // (useful for excluding NFS/CIFS mounts where statfs may hang) + if let Some(ref exclude) = config.filesystem_mount_exclude + && exclude.is_match(&mount_point) + { + continue; + } + + // Call statfs to get filesystem stats (matches Prometheus node_exporter) + let mut stat: StatFs = unsafe { std::mem::zeroed() }; + // `statfs` resolves in this process's mount namespace, so when the host root is + // reachable under a prefix the mount point has to be composed onto it. The + // label keeps the host's own path, so series match a host-side node_exporter. + let rootfs = config.rootfs_path.trim_end_matches('/'); + let statfs_target = if rootfs.is_empty() { + mount_point.clone() + } else { + format!("{}/{}", rootfs, mount_point.trim_start_matches('/')) + }; + + let mount_point_cstr = match std::ffi::CString::new(statfs_target.as_str()) { + Ok(c) => c, + Err(_) => continue, + }; + + let ret = unsafe { statfs(mount_point_cstr.as_ptr(), &mut stat) }; + let statfs_err = std::io::Error::last_os_error(); + if ret != 0 { + // Warn rather than debug: a filesystem that silently disappears from the + // metrics is what an operator needs to know about during an incident, and + // the usual causes (a stale mount, a permission) are actionable. + warn!("statfs({}) failed: {}", statfs_target, statfs_err); + continue; + } + + let labels = vec![ + ("device".to_string(), device), + ("fstype".to_string(), fstype.to_string()), + ("mountpoint".to_string(), mount_point), + ]; + + let block_size = stat.f_bsize as f64; + let total_bytes = stat.f_blocks as f64 * block_size; + let free_bytes = stat.f_bfree as f64 * block_size; + let avail_bytes = stat.f_bavail as f64 * block_size; + + let fs_metrics = [ + ( + "node_filesystem_size_bytes", + total_bytes, + "Filesystem size in bytes", + ), + ( + "node_filesystem_free_bytes", + free_bytes, + "Filesystem free space in bytes", + ), + ( + "node_filesystem_avail_bytes", + avail_bytes, + "Filesystem space available to non-root users in bytes", + ), + ]; + + for (name, value, description) in fs_metrics { + if config.should_include_metric(name) { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: Some("By".to_string()), + description: Some(description.to_string()), + }); + } + } + + // Files (inodes) + if config.should_include_metric("node_filesystem_files") { + metrics.push(CollectedMetric { + name: "node_filesystem_files".to_string(), + value: stat.f_files as f64, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Filesystem total file nodes".to_string()), + }); + } + + if config.should_include_metric("node_filesystem_files_free") { + metrics.push(CollectedMetric { + name: "node_filesystem_files_free".to_string(), + value: stat.f_ffree as f64, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Filesystem free file nodes".to_string()), + }); + } + + // Readonly flag, taken from the mount options above + if config.should_include_metric("node_filesystem_readonly") { + metrics.push(CollectedMetric { + name: "node_filesystem_readonly".to_string(), + value: if read_only { 1.0 } else { 0.0 }, + labels, + metric_type: MetricType::Gauge, + unit: None, + description: Some("Filesystem read-only status".to_string()), + }); + } + } + + metrics + } + + /// Collect uname/system information + pub(super) fn collect_uname(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Get uname info + if !config.should_include_metric("node_uname_info") { + return metrics; + } + + let mut uname_info: libc::utsname = unsafe { std::mem::zeroed() }; + let ret = unsafe { libc::uname(&mut uname_info) }; + + if ret == 0 { + // Convert C strings to Rust strings + let sysname = utsname_field(&uname_info.sysname); + let nodename = utsname_field(&uname_info.nodename); + let release = utsname_field(&uname_info.release); + let version = utsname_field(&uname_info.version); + let machine = utsname_field(&uname_info.machine); + + #[cfg(target_os = "linux")] + let domainname = utsname_field(&uname_info.domainname); + + #[allow(unused_mut)] // domainname is pushed only on Linux + let mut labels = vec![ + ("sysname".to_string(), sysname), + ("nodename".to_string(), nodename), + ("release".to_string(), release), + ("version".to_string(), version), + ("machine".to_string(), machine), + ]; + #[cfg(target_os = "linux")] + labels.push(("domainname".to_string(), domainname)); + + metrics.push(CollectedMetric { + name: "node_uname_info".to_string(), + value: 1.0, + labels, + metric_type: MetricType::Gauge, + unit: None, + description: Some( + "Labeled system information as provided by the uname system call".to_string(), + ), + }); + } + + metrics + } + + /// Collect time metrics (boot time, current time) + /// + /// This is a separate collector from uname, matching Prometheus node_exporter's + /// `time` collector. Boot time originates from `/proc/stat` (the `btime` line) and is + /// grouped here so it can be toggled independently. Note node_exporter emits + /// `node_boot_time_seconds` from its `stat` collector instead. + pub(super) fn collect_time(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Boot time, read from /proc/stat btime. Suppressed entirely while unknown: + // reporting 0 would look like a plausible (1970) boot time. + let boot_time = self.boot_time(); + if boot_time > 0 && config.should_include_metric("node_boot_time_seconds") { + metrics.push(CollectedMetric { + name: "node_boot_time_seconds".to_string(), + value: boot_time as f64, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("s".to_string()), + description: Some("Node boot time in seconds since epoch".to_string()), + }); + } + + // Current time. This reads the clock independently of the OTLP + // data-point timestamp set later in convert_to_otlp_metrics, so the + // two may differ by the time it takes to run the remaining collectors. + // Prometheus node_exporter has the same inherent skew. + if config.should_include_metric("node_time_seconds") { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); + + metrics.push(CollectedMetric { + name: "node_time_seconds".to_string(), + value: now, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("s".to_string()), + description: Some("System time in seconds since epoch".to_string()), + }); + } + + metrics + } + + /// Collect kernel stat counters from /proc/stat + /// + /// Includes forks, context switches, interrupts, and procs running/blocked. + /// The per-CPU `cpu*` lines are handled by the cpu collector; `softirq` and the + /// per-interrupt vector of `intr` are not currently exposed. + pub(super) fn collect_stat( + &self, + config: &NodeMetricsConfig, + stat_content: Option<&str>, + ) -> Vec { + let mut metrics = Vec::new(); + + let content = match stat_content { + Some(c) => c, + None => return metrics, + }; + + for line in content.lines() { + if let Some(rest) = line.strip_prefix("processes ") { + if config.should_include_metric("node_forks_total") + && let Ok(value) = rest.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_forks_total".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Counter, + unit: None, + description: Some("Total number of forks".to_string()), + }); + } + } else if let Some(rest) = line.strip_prefix("ctxt ") { + if config.should_include_metric("node_context_switches_total") + && let Ok(value) = rest.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_context_switches_total".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Counter, + unit: None, + description: Some("Total number of context switches".to_string()), + }); + } + } else if let Some(rest) = line.strip_prefix("intr ") { + // Test the metric filter inside the arm, like the sibling arms do, so that + // control flow through this chain does not depend on the filter config. + // The first number is total interrupts. Skip the metric entirely if it + // cannot be parsed — a fabricated 0 reads as a counter reset downstream. + if config.should_include_metric("node_intr_total") + && let Some(total) = rest + .split_whitespace() + .next() + .and_then(|s| s.parse::().ok()) + { + metrics.push(CollectedMetric { + name: "node_intr_total".to_string(), + value: total, + labels: Vec::new(), + metric_type: MetricType::Counter, + unit: Some("{interrupts}".to_string()), + description: Some("Total number of interrupts serviced".to_string()), + }); + } + } else if let Some(rest) = line.strip_prefix("procs_running ") { + if config.should_include_metric("node_procs_running") + && let Ok(value) = rest.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_procs_running".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Number of processes in runnable state".to_string()), + }); + } + } else if let Some(rest) = line.strip_prefix("procs_blocked ") { + // Collapsing this into the chain condition would make control flow through + // the whole chain depend on the metric filter; only the final arm trips the + // lint, and all five arms deliberately test the filter inside the arm. + #[allow(clippy::collapsible_if)] + if config.should_include_metric("node_procs_blocked") + && let Ok(value) = rest.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_procs_blocked".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some( + "Number of processes blocked waiting for I/O".to_string(), + ), + }); + } + } + } + + metrics + } + + /// Collect process metrics: kernel limits from /proc/sys/kernel/ + pub(super) fn collect_processes(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Kernel limits (single-file reads, no per-PID scanning) + if config.should_include_metric("node_processes_max_threads") { + let path = format!("{}/sys/kernel/threads-max", self.procfs_path); + if let Ok(content) = fs::read_to_string(&path) + && let Ok(value) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_processes_max_threads".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Limit of threads in the system".to_string()), + }); + } + } + + if config.should_include_metric("node_processes_max_processes") { + let path = format!("{}/sys/kernel/pid_max", self.procfs_path); + if let Ok(content) = fs::read_to_string(&path) + && let Ok(value) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_processes_max_processes".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Limit of PID values in the system".to_string()), + }); + } + } + + metrics + } + + /// Collect disk I/O statistics from /proc/diskstats + pub(super) fn collect_diskstats(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/diskstats", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + for line in content.lines() { + let fields: Vec<&str> = line.split_whitespace().collect(); + if fields.len() < 14 { + continue; + } + + let device = fields[2]; + + // Skip virtual devices, matching node_exporter's default exclusion of + // `^(z?ram|loop|fd)\d+$` — anchored on a trailing device number so that a + // real disk sharing one of these prefixes is not dropped. + if is_virtual_disk(device) { + continue; + } + + // Skip partitions: sda1, nvme0n1p1, etc. + // Keep whole disks: sda, nvme0n1, vda, xvda + if is_partition(device) { + continue; + } + + let labels = vec![("device".to_string(), device.to_string())]; + + // Parse helper: any field that cannot be parsed causes the entire device to + // be skipped, rather than emitting a misleading zero value or a partial set + // of metrics for that device. + let parse = |idx: usize| -> Option { fields[idx].parse().ok() }; + + // Parse core values - skip entire device on any parse error + let Some(rd_ios) = parse(3) else { continue }; + let Some(rd_merges) = parse(4) else { continue }; + let Some(rd_sectors) = parse(5) else { continue }; + let Some(rd_ticks) = parse(6) else { continue }; + let Some(wr_ios) = parse(7) else { continue }; + let Some(wr_merges) = parse(8) else { continue }; + let Some(wr_sectors) = parse(9) else { continue }; + let Some(wr_ticks) = parse(10) else { continue }; + let Some(ios_in_progress) = parse(11) else { + continue; + }; + let Some(io_ticks) = parse(12) else { continue }; + let Some(time_in_queue) = parse(13) else { + continue; + }; + + // Parse the optional extended field groups up front, so that a corrupt field + // skips the whole device instead of emitting the core metrics and then + // silently dropping the rest of the groups. + let discard = if fields.len() >= 18 { + match (parse(14), parse(15), parse(16), parse(17)) { + (Some(a), Some(b), Some(c), Some(d)) => Some((a, b, c, d)), + _ => continue, + } + } else { + None + }; + let flush = if fields.len() >= 20 { + match (parse(18), parse(19)) { + (Some(a), Some(b)) => Some((a, b)), + _ => continue, + } + } else { + None + }; + + // Convert sectors to bytes (sector = 512 bytes) + let rd_bytes = rd_sectors * 512.0; + let wr_bytes = wr_sectors * 512.0; + + // Convert milliseconds to seconds + let rd_time_seconds = rd_ticks / 1000.0; + let wr_time_seconds = wr_ticks / 1000.0; + let io_time_seconds = io_ticks / 1000.0; + let time_in_queue_seconds = time_in_queue / 1000.0; + + let disk_metrics = [ + ( + "node_disk_reads_completed_total", + rd_ios, + MetricType::Counter, + "{operations}", + "The total number of reads completed successfully", + ), + ( + "node_disk_reads_merged_total", + rd_merges, + MetricType::Counter, + "{operations}", + "The total number of reads merged", + ), + ( + "node_disk_read_bytes_total", + rd_bytes, + MetricType::Counter, + "By", + "The total number of bytes read successfully", + ), + ( + "node_disk_read_time_seconds_total", + rd_time_seconds, + MetricType::Counter, + "s", + "The total number of seconds spent by all reads", + ), + ( + "node_disk_writes_completed_total", + wr_ios, + MetricType::Counter, + "{operations}", + "The total number of writes completed successfully", + ), + ( + "node_disk_writes_merged_total", + wr_merges, + MetricType::Counter, + "{operations}", + "The total number of writes merged", + ), + ( + "node_disk_written_bytes_total", + wr_bytes, + MetricType::Counter, + "By", + "The total number of bytes written successfully", + ), + ( + "node_disk_write_time_seconds_total", + wr_time_seconds, + MetricType::Counter, + "s", + "The total number of seconds spent by all writes", + ), + ( + "node_disk_io_now", + ios_in_progress, + MetricType::Gauge, + "{operations}", + "The number of I/Os currently in progress", + ), + ( + "node_disk_io_time_seconds_total", + io_time_seconds, + MetricType::Counter, + "s", + "Total seconds spent doing I/Os", + ), + ( + "node_disk_io_time_weighted_seconds_total", + time_in_queue_seconds, + MetricType::Counter, + "s", + "The weighted number of seconds spent doing I/Os", + ), + ]; + + for (name, value, metric_type, unit, description) in disk_metrics { + if config.should_include_metric(name) { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type, + unit: Some(unit.to_string()), + description: Some(description.to_string()), + }); + } + } + + // Discard metrics (kernel 4.18+, fields 14-17) + if let Some((discard_ios, discard_merges, discard_sectors, discard_ticks)) = discard { + let discard_time_seconds = discard_ticks / 1000.0; + + let discard_metrics = [ + ( + "node_disk_discards_completed_total", + discard_ios, + "{operations}", + "The total number of discards completed successfully", + ), + ( + "node_disk_discards_merged_total", + discard_merges, + "{operations}", + "The total number of discards merged", + ), + ( + "node_disk_discarded_sectors_total", + discard_sectors, + "{sectors}", + "The total number of sectors discarded successfully", + ), + ( + "node_disk_discard_time_seconds_total", + discard_time_seconds, + "s", + "The total number of seconds spent discarding", + ), + ]; + + for (name, value, unit, description) in discard_metrics { + if config.should_include_metric(name) { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type: MetricType::Counter, + unit: Some(unit.to_string()), + description: Some(description.to_string()), + }); + } + } + } + + // Flush metrics (kernel 5.5+, fields 18-19) + if let Some((flush_ios, flush_ticks)) = flush { + let flush_time_seconds = flush_ticks / 1000.0; + + let flush_metrics = [ + ( + "node_disk_flush_requests_total", + flush_ios, + "{operations}", + "The total number of flush requests completed successfully", + ), + ( + "node_disk_flush_requests_time_seconds_total", + flush_time_seconds, + "s", + "The total number of seconds spent flushing", + ), + ]; + + for (name, value, unit, description) in flush_metrics { + if config.should_include_metric(name) { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type: MetricType::Counter, + unit: Some(unit.to_string()), + description: Some(description.to_string()), + }); + } + } + } + } + + metrics + } + + /// Collect virtual memory statistics from /proc/vmstat + pub(super) fn collect_vmstat(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/vmstat", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + for line in content.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 2 { + continue; + } + + let field_name = parts[0]; + let value: f64 = match parts[1].parse() { + Ok(v) => v, + Err(_) => continue, + }; + + // Convert field name to metric name: "pgfault" -> "node_vmstat_pgfault" + let metric_name = format!("node_vmstat_{}", field_name); + + if config.should_include_metric(&metric_name) { + // Most vmstat fields are counters, but nr_* fields are generally gauges + // (they represent current page counts, not cumulative events). + // The workingset_* family includes both gauges (workingset_nodes) + // and counters (workingset_refault_*). We treat nr_* as gauges except + // for the cumulative event counters that also carry the nr_ prefix; for + // workingset_* we whitelist the known gauge. + // numa_* fields (numa_hit, numa_miss, etc.) are cumulative counters. + let is_nr_counter = matches!( + field_name, + "nr_dirtied" + | "nr_written" + | "nr_throttled_written" + | "nr_vmscan_write" + | "nr_vmscan_immediate_reclaim" + | "nr_foll_pin_acquired" + | "nr_foll_pin_released" + ) || field_name.starts_with("nr_tlb_"); + + let metric_type = if !is_nr_counter + && (field_name.starts_with("nr_") || field_name == "workingset_nodes") + { + MetricType::Gauge + } else { + MetricType::Counter + }; + + metrics.push(CollectedMetric { + name: metric_name, + value, + labels: Vec::new(), + metric_type, + unit: None, + description: None, + }); + } + } + + metrics + } + + /// Collect network statistics from /proc/net/netstat and /proc/net/snmp. + /// + /// Both files share the same alternating header/value line format. + /// `/proc/net/netstat` contains TcpExt, IpExt, MPTcpExt counters. + /// `/proc/net/snmp` contains Tcp, Udp, Ip, Icmp, etc. — including the + /// commonly monitored gauge `CurrEstab`. + pub(super) fn collect_netstat(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Known gauge fields in /proc/net/snmp (all others are counters) + let snmp_gauges = [ + "CurrEstab", + "RtoAlgorithm", + "RtoMin", + "RtoMax", + "MaxConn", + "Forwarding", + "DefaultTTL", + ]; + + for subpath in ["net/netstat", "net/snmp"] { + let path = format!("{}/{}", self.procfs_path, subpath); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + continue; + } + }; + + // File format: alternating lines of "Protocol: key1 key2 ..." and "Protocol: val1 val2 ..." + let lines: Vec<&str> = content.lines().collect(); + let mut i = 0; + + while i + 1 < lines.len() { + let header_line = lines[i]; + let value_line = lines[i + 1]; + + let header_parts: Vec<&str> = header_line.split_whitespace().collect(); + let value_parts: Vec<&str> = value_line.split_whitespace().collect(); + + // Advance by one, not two, when the pair does not line up: an unexpected + // or missing line would otherwise desynchronise every remaining protocol + // in the file instead of resynchronising on the next header. + if header_parts.is_empty() || value_parts.is_empty() { + debug!("Skipping unpaired line {} in {}", i, path); + i += 1; + continue; + } + + // Preserve original casing to match Prometheus node_exporter + // conventions (e.g. "Tcp", "Udp", "TcpExt", "IpExt"). + let protocol = header_parts[0].trim_end_matches(':'); + + let value_protocol = value_parts[0].trim_end_matches(':'); + if protocol != value_protocol { + debug!( + "Skipping line {} in {}: expected values for {}, found {}", + i, path, protocol, value_protocol + ); + i += 1; + continue; + } + + for j in 1..header_parts.len().min(value_parts.len()) { + let key = header_parts[j]; + let value: f64 = match value_parts[j].parse() { + Ok(v) => v, + Err(_) => continue, + }; + + let metric_name = format!("node_netstat_{}_{}", protocol, key); + + if config.should_include_metric(&metric_name) { + let metric_type = if snmp_gauges.contains(&key) { + MetricType::Gauge + } else { + MetricType::Counter + }; + + metrics.push(CollectedMetric { + name: metric_name, + value, + labels: Vec::new(), + metric_type, + unit: None, + description: None, + }); + } + } + + i += 2; + } + } + + metrics + } + + /// Collect socket statistics from /proc/net/sockstat and /proc/net/sockstat6 + pub(super) fn collect_sockstat(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + let page_size = self.page_size; + + for subpath in ["net/sockstat", "net/sockstat6"] { + let path = format!("{}/{}", self.procfs_path, subpath); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + continue; + } + }; + + // Preserve the original casing to match Prometheus node_exporter + // conventions (e.g. "TCP", "UDP", "FRAG", "TCP6"). + for line in content.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 3 { + continue; + } + + // Format: "protocol: key1 val1 key2 val2 ..." + let protocol = parts[0].trim_end_matches(':'); + + // Parse key-value pairs + let mut j = 1; + while j + 1 < parts.len() { + let key = parts[j]; + let raw_value: f64 = match parts[j + 1].parse() { + Ok(v) => v, + Err(_) => { + j += 2; + continue; + } + }; + + // Emit the raw metric + let metric_name = format!("node_sockstat_{}_{}", protocol, key); + if config.should_include_metric(&metric_name) { + metrics.push(CollectedMetric { + name: metric_name, + value: raw_value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: None, + }); + } + + // For "mem" fields, also emit a _bytes metric (pages -> bytes) + if key == "mem" { + let bytes_name = format!("node_sockstat_{}_{}_bytes", protocol, key); + if config.should_include_metric(&bytes_name) { + metrics.push(CollectedMetric { + name: bytes_name, + value: raw_value * page_size, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("By".to_string()), + description: None, + }); + } + } + + j += 2; + } + } + } + + metrics + } + + /// Collect file descriptor statistics from /proc/sys/fs/file-nr + pub(super) fn collect_filefd(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/sys/fs/file-nr", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + // Format: "allocated_fds free_fds max_fds" + let parts: Vec<&str> = content.split_whitespace().collect(); + if parts.len() < 3 { + return metrics; + } + + // Skip a field that does not parse rather than reporting a misleading 0 + // (0 allocated descriptors is a plausible-looking but impossible value). + // parts[1] is "free" — always 0 on kernels ≥2.6, not exposed by node_exporter + if config.should_include_metric("node_filefd_allocated") + && let Ok(allocated) = parts[0].parse::() + { + metrics.push(CollectedMetric { + name: "node_filefd_allocated".to_string(), + value: allocated, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("{file_descriptors}".to_string()), + description: Some("Number of allocated file descriptors".to_string()), + }); + } + + if config.should_include_metric("node_filefd_maximum") + && let Ok(max) = parts[2].parse::() + { + metrics.push(CollectedMetric { + name: "node_filefd_maximum".to_string(), + value: max, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("{file_descriptors}".to_string()), + description: Some("Maximum number of file descriptors".to_string()), + }); + } + + metrics + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::receivers::node_metrics::collector::test_support::*; + use crate::receivers::node_metrics::config::Collector; + use tracing_test::traced_test; + + #[test] + fn test_collect_cpu() { + let tmp = tempfile::tempdir().unwrap(); + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let tck = collector.clk_tck; + + let metrics = collector.collect_cpu(&config_for(vec![Collector::Cpu]), Some(STAT_FIXTURE)); + + // The aggregate "cpu" line yields no data point: only per-core lines are + // reported, matching Prometheus node_exporter. + assert_eq!(label_values(&metrics, "cpu"), ["0", "1", "2"]); + + let user0 = find_one( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "0"), ("mode", "user")], + ); + assert_eq!(user0.value, 1000.0 / tck); + assert_eq!(user0.metric_type, MetricType::Counter); + assert_eq!(user0.unit.as_deref(), Some("s")); + + assert_eq!( + find_one( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "1"), ("mode", "idle")] + ) + .value, + 800_000.0 / tck + ); + assert_eq!( + find_one( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "0"), ("mode", "steal")] + ) + .value, + 8.0 / tck + ); + + // The guest fields go to their own metric, not to node_cpu_seconds_total. + assert_eq!( + find_one( + &metrics, + "node_cpu_guest_seconds_total", + &[("cpu", "0"), ("mode", "user")] + ) + .value, + 900.0 / tck + ); + assert_eq!( + find_one( + &metrics, + "node_cpu_guest_seconds_total", + &[("cpu", "0"), ("mode", "nice")] + ) + .value, + 10.0 / tck + ); + assert!( + matching( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "0"), ("mode", "guest")] + ) + .is_empty() + ); + + // A truncated line reports exactly the fields it has: absent columns are + // skipped rather than reported as a counter-resetting zero. + let cpu2_modes: Vec<&str> = metrics + .iter() + .filter(|m| label(m, "cpu") == "2") + .map(|m| label(m, "mode")) + .collect(); + assert_eq!(cpu2_modes, ["user", "nice", "system"]); + assert!(matching(&metrics, "node_cpu_guest_seconds_total", &[("cpu", "2")]).is_empty()); + } + + #[test] + fn test_collect_meminfo() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "meminfo", + "\ +MemTotal: 16 kB +MemFree: 8192 kB +Active(anon): 4 kB +HugePages_Total: 0 +Broken: notanumber kB +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_meminfo(&config_for(vec![Collector::Memory])); + + let total = find_one(&metrics, "node_memory_MemTotal_bytes", &[]); + assert_eq!(total.value, 16384.0); + assert_eq!(total.unit.as_deref(), Some("By")); + assert_eq!(total.metric_type, MetricType::Gauge); + + // Parenthesised suffixes are normalised into the metric name. + assert_eq!( + find_one(&metrics, "node_memory_Active_anon_bytes", &[]).value, + 4096.0 + ); + + // A field without the "kB" unit keeps its raw value, gets no _bytes + // suffix and carries no OTLP unit. + let hugepages = find_one(&metrics, "node_memory_HugePages_Total", &[]); + assert_eq!(hugepages.value, 0.0); + assert_eq!(hugepages.unit, None); + + // The unparseable line is skipped entirely, leaving only the four good ones. + assert!( + !metrics.iter().any(|m| m.name.contains("Broken")), + "garbage line should be skipped, got {:?}", + metrics.iter().map(|m| &m.name).collect::>() + ); + assert_eq!(metrics.len(), 4); + } + + #[test] + fn test_collect_netdev() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "net/dev", + "\ +Inter-| Receive | Transmit + face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed + eth0: 123456789 98765 1 2 3 4 5 6 987654321 87654 7 8 9 10 11 12 + lo: 1000 10 0 0 0 0 0 0 1000 10 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_netdev(&config_for(vec![Collector::Network])); + + // Only eth0 is reported: the two header lines carry no counters and the + // 10-column "lo" line is too short to be trusted. + assert_eq!(label_values(&metrics, "device"), ["eth0"]); + assert_eq!(metrics.len(), 16); + + let rx_bytes = find_one( + &metrics, + "node_network_receive_bytes_total", + &[("device", "eth0")], + ); + assert_eq!(rx_bytes.value, 123_456_789.0); + assert_eq!(rx_bytes.unit.as_deref(), Some("By")); + assert_eq!(rx_bytes.metric_type, MetricType::Counter); + + // Transmit starts at column 9, not wherever the receive block happened to end. + assert_eq!( + find_one( + &metrics, + "node_network_transmit_bytes_total", + &[("device", "eth0")] + ) + .value, + 987_654_321.0 + ); + assert_eq!( + find_one( + &metrics, + "node_network_receive_multicast_total", + &[("device", "eth0")] + ) + .value, + 6.0 + ); + assert_eq!( + find_one( + &metrics, + "node_network_transmit_compressed_total", + &[("device", "eth0")] + ) + .value, + 12.0 + ); + } + + #[test] + fn test_collect_diskstats() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "diskstats", + "\ + 8 0 sda 100 10 2000 3000 200 20 4000 5000 0 6000 7000 + 8 1 sda1 50 5 1000 1500 100 10 2000 2500 0 3000 3500 + 8 16 sdb 100 10 abc 3000 200 20 4000 5000 0 6000 7000 + 7 0 loop0 1 0 8 2 0 0 0 0 0 4 2 + 252 0 zram0 5 0 40 1 6 0 48 2 0 3 3 + 253 0 vda 10 1 200 300 20 2 400 500 0 600 700 1 2 3 400 + 259 0 nvme0n1 1000 100 20000 30000 2000 200 40000 50000 0 60000 70000 5 6 7 800 9 1000 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_diskstats(&config_for(vec![Collector::Diskstats])); + + // Partitions (sda1), virtual devices (loop0, zram0) and the device with a + // corrupt core field (sdb) contribute nothing at all. + assert_eq!(label_values(&metrics, "device"), ["nvme0n1", "sda", "vda"]); + + let sda = [("device", "sda")]; + + // Sectors are 512 bytes; the tick fields are milliseconds. + assert_eq!( + find_one(&metrics, "node_disk_read_bytes_total", &sda).value, + 2000.0 * 512.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_written_bytes_total", &sda).value, + 4000.0 * 512.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_read_time_seconds_total", &sda).value, + 3.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_io_time_weighted_seconds_total", &sda).value, + 7.0 + ); + + // In-flight I/O is a point-in-time reading, unlike its siblings. + let io_now = find_one(&metrics, "node_disk_io_now", &sda); + assert_eq!(io_now.value, 0.0); + assert_eq!(io_now.metric_type, MetricType::Gauge); + assert_eq!( + find_one(&metrics, "node_disk_reads_completed_total", &sda).metric_type, + MetricType::Counter + ); + assert_eq!( + find_one(&metrics, "node_disk_io_time_seconds_total", &sda).metric_type, + MetricType::Counter + ); + + // 14 fields: no discard group and no flush group. + assert!(matching(&metrics, "node_disk_discards_completed_total", &sda).is_empty()); + assert!(matching(&metrics, "node_disk_flush_requests_total", &sda).is_empty()); + + // 18 fields: discard only. + let vda = [("device", "vda")]; + assert_eq!( + find_one(&metrics, "node_disk_discarded_sectors_total", &vda).value, + 3.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_discard_time_seconds_total", &vda).value, + 0.4 + ); + assert!(matching(&metrics, "node_disk_flush_requests_total", &vda).is_empty()); + + // 20 fields: discard and flush. + let nvme = [("device", "nvme0n1")]; + assert_eq!( + find_one(&metrics, "node_disk_discards_completed_total", &nvme).value, + 5.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_flush_requests_total", &nvme).value, + 9.0 + ); + assert_eq!( + find_one( + &metrics, + "node_disk_flush_requests_time_seconds_total", + &nvme + ) + .value, + 1.0 + ); + } + + #[test] + fn test_collect_stat_and_processes() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "stat", STAT_FIXTURE); + write(tmp.path(), "sys/kernel/threads-max", "126988\n"); + write(tmp.path(), "sys/kernel/pid_max", "4194304\n"); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let config = config_for(vec![Collector::Stat, Collector::Processes]); + + let stat = std::fs::read_to_string(tmp.path().join("stat")).unwrap(); + let mut metrics = collector.collect_stat(&config, Some(&stat)); + metrics.extend(collector.collect_processes(&config)); + + // Only the first number on the "intr" line is the total; the per-vector + // counts that follow are not summed into it. + let intr = find_one(&metrics, "node_intr_total", &[]); + assert_eq!(intr.value, 1234.0); + assert_eq!(intr.metric_type, MetricType::Counter); + assert_eq!(intr.unit.as_deref(), Some("{interrupts}")); + + let forks = find_one(&metrics, "node_forks_total", &[]); + assert_eq!(forks.value, 4242.0); + assert_eq!(forks.metric_type, MetricType::Counter); + + let ctxt = find_one(&metrics, "node_context_switches_total", &[]); + assert_eq!(ctxt.value, 987_654_321.0); + assert_eq!(ctxt.metric_type, MetricType::Counter); + + // Process counts are instantaneous readings, not cumulative. + let running = find_one(&metrics, "node_procs_running", &[]); + assert_eq!(running.value, 3.0); + assert_eq!(running.metric_type, MetricType::Gauge); + + let blocked = find_one(&metrics, "node_procs_blocked", &[]); + assert_eq!(blocked.value, 1.0); + assert_eq!(blocked.metric_type, MetricType::Gauge); + + // The two limits come from /proc/sys/kernel, not from /proc/stat. + assert_eq!( + find_one(&metrics, "node_processes_max_threads", &[]).value, + 126_988.0 + ); + assert_eq!( + find_one(&metrics, "node_processes_max_processes", &[]).value, + 4_194_304.0 + ); + } + + #[test] + fn test_collect_sockstat() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "net/sockstat", + "\ +sockets: used 100 +TCP: inuse 1 orphan 0 tw 0 alloc 2 mem 5 +UDP: inuse 3 mem 1 +", + ); + write( + tmp.path(), + "net/sockstat6", + "\ +TCP6: inuse 2 +UDP6: inuse 1 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let page_size = collector.page_size; + let metrics = collector.collect_sockstat(&config_for(vec![Collector::Sockstat])); + + assert_eq!( + find_one(&metrics, "node_sockstat_sockets_used", &[]).value, + 100.0 + ); + assert_eq!( + find_one(&metrics, "node_sockstat_TCP_inuse", &[]).value, + 1.0 + ); + assert_eq!( + find_one(&metrics, "node_sockstat_TCP_alloc", &[]).value, + 2.0 + ); + + // "mem" is a page count; the derived _bytes metric scales it by the page size. + assert_eq!(find_one(&metrics, "node_sockstat_TCP_mem", &[]).value, 5.0); + let mem_bytes = find_one(&metrics, "node_sockstat_TCP_mem_bytes", &[]); + assert_eq!(mem_bytes.value, 5.0 * page_size); + assert_eq!(mem_bytes.unit.as_deref(), Some("By")); + assert_eq!(mem_bytes.metric_type, MetricType::Gauge); + + // The IPv6 file contributes its own protocol names. + assert_eq!( + find_one(&metrics, "node_sockstat_TCP6_inuse", &[]).value, + 2.0 + ); + assert_eq!( + find_one(&metrics, "node_sockstat_UDP6_inuse", &[]).value, + 1.0 + ); + assert!(matching(&metrics, "node_sockstat_TCP6_mem_bytes", &[]).is_empty()); + } + + #[test] + fn test_collect_filefd() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "sys/fs/file-nr", + "1234\t0\t9223372036854775807\n", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_filefd(&config_for(vec![Collector::Filefd])); + + let allocated = find_one(&metrics, "node_filefd_allocated", &[]); + assert_eq!(allocated.value, 1234.0); + assert_eq!(allocated.metric_type, MetricType::Gauge); + assert_eq!(allocated.unit.as_deref(), Some("{file_descriptors}")); + assert_eq!( + find_one(&metrics, "node_filefd_maximum", &[]).value, + 9_223_372_036_854_775_807f64 + ); + + // A field that does not parse is skipped, not reported as an impossible 0. + let broken = tempfile::tempdir().unwrap(); + write(broken.path(), "sys/fs/file-nr", "notanumber\t0\t100\n"); + let collector = SystemCollector::new(broken.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_filefd(&config_for(vec![Collector::Filefd])); + assert!(matching(&metrics, "node_filefd_allocated", &[]).is_empty()); + assert_eq!(find_one(&metrics, "node_filefd_maximum", &[]).value, 100.0); + } + + #[test] + fn test_collect_filesystem() { + let tmp = tempfile::tempdir().unwrap(); + let data = tmp.path().join("data"); + let excluded = tmp.path().join("excluded"); + let labelled = tmp.path().join("labelled"); + let merged = tmp.path().join("merged"); + let remote = tmp.path().join("remote"); + let ubi = tmp.path().join("ubi"); + let readonly_dir = tmp.path().join("readonly"); + for dir in [ + &data, + &excluded, + &labelled, + &merged, + &remote, + &ubi, + &readonly_dir, + ] { + std::fs::create_dir_all(dir).unwrap(); + } + let (data, excluded, labelled, merged, remote, ubi, readonly) = ( + data.to_str().unwrap(), + excluded.to_str().unwrap(), + labelled.to_str().unwrap(), + merged.to_str().unwrap(), + remote.to_str().unwrap(), + ubi.to_str().unwrap(), + readonly_dir.to_str().unwrap(), + ); + + // The mount points are real directories so the collector's statfs(2) call + // succeeds; only the fstype and device columns are synthetic. + let mounts = format!( + "/dev/sda2 {data} ext4 rw,relatime 0 0\n\ + /dev/sda2 {data} ext4 rw,relatime 0 0\n\ + proc {data} proc rw,nosuid,nodev,noexec,relatime 0 0\n\ + sysfs {data} sysfs rw,nosuid,nodev,noexec,relatime 0 0\n\ + fileserver:/exports/home {data} nfs4 rw,relatime 0 0\n\ + /dev/sdb1 {excluded} ext4 rw,relatime 0 0\n\ + /dev/disk/by-label/My\\040Disk {labelled} ext4 rw,relatime 0 0\n\ + mergerfs {merged} fuse.mergerfs rw,relatime 0 0\n\ + user@fileserver:/srv {remote} fuse.sshfs rw,relatime 0 0\n\ + ubi0:rootfs {ubi} ubifs rw,relatime 0 0\n\ + /dev/sdc1 {readonly} ext4 ro,relatime 0 0\n" + ); + write(tmp.path(), "mounts", &mounts); + + let config = NodeMetricsConfig { + collectors: vec![Collector::Filesystem], + filesystem_mount_exclude: Some( + regex::Regex::new(&format!("^{}$", regex::escape(excluded))).unwrap(), + ), + ..Default::default() + }; + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_filesystem(&config); + + // proc/sysfs are dropped by fstype, nfs4 and fuse.sshfs as remote filesystems + // whose statfs can block, the repeated (device, mountpoint) pair as a duplicate, + // and `excluded` by filesystem_mount_exclude. + assert_eq!( + label_values(&metrics, "fstype"), + ["ext4", "fuse.mergerfs", "ubifs"] + ); + // The survivors share the temp-dir prefix, so they sort by directory name. + assert_eq!( + label_values(&metrics, "mountpoint"), + [ + data.to_string(), + labelled.to_string(), + merged.to_string(), + readonly.to_string(), + ubi.to_string() + ] + ); + + let size = find_one( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", data)], + ); + assert_eq!(size.value, statfs_size_bytes(data)); + assert_eq!(size.unit.as_deref(), Some("By")); + assert_eq!(size.metric_type, MetricType::Gauge); + assert_eq!(label(size, "device"), "/dev/sda2"); + assert_eq!(label(size, "fstype"), "ext4"); + + // The remaining statfs-derived metrics are reported for the same mount. + for name in [ + "node_filesystem_free_bytes", + "node_filesystem_avail_bytes", + "node_filesystem_files", + "node_filesystem_files_free", + "node_filesystem_readonly", + ] { + find_one(&metrics, name, &[("mountpoint", data)]); + } + + // The read-only state comes from the mount options, since `statfs.f_flags` exists + // only on the BSDs and not on Linux. + assert_eq!( + find_one( + &metrics, + "node_filesystem_readonly", + &[("mountpoint", data)] + ) + .value, + 0.0 + ); + assert_eq!( + find_one( + &metrics, + "node_filesystem_readonly", + &[("mountpoint", readonly)] + ) + .value, + 1.0 + ); + + // Octal escapes in the device column are decoded before labelling. + assert_eq!( + label( + find_one( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", labelled)] + ), + "device" + ), + "/dev/disk/by-label/My Disk" + ); + + // FUSE mounts are named by subtype, and only the network-backed subtypes are + // excluded: a local FUSE filesystem is as worth reporting as any other, so the + // exclusion cannot be a blanket match on the "fuse" prefix. + let local_fuse = find_one( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", merged)], + ); + assert_eq!(label(local_fuse, "fstype"), "fuse.mergerfs"); + assert_eq!(label(local_fuse, "device"), "mergerfs"); + assert_eq!(local_fuse.value, statfs_size_bytes(merged)); + + // ...while the network-backed subtype is skipped, because statfs on a mount + // whose server has gone away blocks uninterruptibly on a shared blocking thread. + assert!( + matching( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", remote)] + ) + .is_empty(), + "a remote FUSE mount must not be scraped" + ); + + // ubifs is the on-device filesystem of the embedded targets this receiver runs + // on, so losing it would be the worst possible regression. + let flash = find_one( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", ubi)], + ); + assert_eq!(label(flash, "fstype"), "ubifs"); + assert_eq!(label(flash, "device"), "ubi0:rootfs"); + assert_eq!(flash.value, statfs_size_bytes(ubi)); + assert_eq!(flash.unit.as_deref(), Some("By")); + } + + /// The mount table comes from PID 1, not from `self`: with a bind-mounted host procfs + /// the `self` table is the *reader's* namespace, which would report a container's own + /// bind mounts labelled as if they were the host's filesystems. + #[test] + fn test_collect_filesystem_prefers_pid1_mount_table() { + let procfs = tempfile::tempdir().unwrap(); + let root = tempfile::tempdir().unwrap(); + let host_mount = root.path().join("data"); + std::fs::create_dir_all(&host_mount).unwrap(); + std::fs::create_dir_all(procfs.path().join("1")).unwrap(); + + // What PID 1 sees (the host) versus what this process sees (a container). + write( + &procfs.path().join("1"), + "mounts", + "/dev/sda1 /data ext4 rw 0 0\n", + ); + write( + procfs.path(), + "mounts", + "/dev/sdb1 /container-only ext4 rw 0 0\n", + ); + + let collector = SystemCollector::new(procfs.path().to_str().unwrap(), "/sys"); + let config = NodeMetricsConfig { + collectors: vec![Collector::Filesystem], + procfs_path: procfs.path().to_str().unwrap().to_string(), + // The host root is reachable under this prefix, so statfs is composed onto it + rootfs_path: root.path().to_str().unwrap().to_string(), + ..Default::default() + }; + + let metrics = collector.collect_filesystem(&config); + + // The host's mount is reported, labelled with the host's own path... + let size = find_one( + &metrics, + "node_filesystem_size_bytes", + &[ + ("device", "/dev/sda1"), + ("fstype", "ext4"), + ("mountpoint", "/data"), + ], + ); + assert!(size.value > 0.0); + // ...and the container's own table is not consulted. + assert!( + matching( + &metrics, + "node_filesystem_size_bytes", + &[("device", "/dev/sdb1")] + ) + .is_empty(), + "the self mount table must not be used when PID 1's is readable" + ); + } + + /// The two halves of the mount-point exclusion: the kernel pseudo-filesystem paths + /// are excluded including the mount point itself, while the container/runtime paths + /// are excluded only *below* it — a dedicated volume mounted exactly at + /// `/var/lib/docker` is a real filesystem, and the one most likely to fill up. + /// + /// Those paths are absolute, so a fixture cannot place a real directory at them and + /// statfs will usually fail. What the test can observe is whether the collector got + /// as far as calling statfs at all: an excluded mount point is skipped before that, + /// and so logs no statfs failure and produces no metric. + #[traced_test] + #[test] + fn test_collect_filesystem_mount_exclusion_boundaries() { + let tmp = tempfile::tempdir().unwrap(); + + // Every line uses a real, non-ignored fstype, so the only rule that can drop + // any of them is the mount-point exclusion under test. + write( + tmp.path(), + "mounts", + "\ +/dev/sda1 /dev ext4 rw,relatime 0 0 +/dev/sda2 /dev/shm ext4 rw,relatime 0 0 +/dev/sda3 /proc ext4 rw,relatime 0 0 +/dev/sda4 /sys ext4 rw,relatime 0 0 +/dev/sda5 /var/lib/docker ext4 rw,relatime 0 0 +/dev/sda6 /var/lib/docker/volumes/vol1 ext4 rw,relatime 0 0 +/dev/sda7 /var/lib/containers ext4 rw,relatime 0 0 +/dev/sda8 /var/lib/containers/storage ext4 rw,relatime 0 0 +/dev/sda9 /run/user/1000 ext4 rw,relatime 0 0 +/dev/sda10 /devices ext4 rw,relatime 0 0 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_filesystem(&config_for(vec![Collector::Filesystem])); + + // True when the mount point reached the statfs call: either it succeeded and + // produced a metric, or it failed and said so. + let reached_statfs = |mount_point: &str| { + !matching( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", mount_point)], + ) + .is_empty() + || logs_contain(&format!("statfs({}) failed", mount_point)) + }; + + // Kernel pseudo-filesystems: the mount point itself is excluded, not just what + // is below it. `/dev` exists on every unix, so it would otherwise be reported. + for excluded in ["/dev", "/dev/shm", "/proc", "/sys"] { + assert!( + !reached_statfs(excluded), + "{} must be excluded before statfs is called", + excluded + ); + } + + // Container and runtime paths: excluded only below the mount point. + for excluded in [ + "/var/lib/docker/volumes/vol1", + "/var/lib/containers/storage", + "/run/user/1000", + ] { + assert!( + !reached_statfs(excluded), + "{} is below an excluded path and must be skipped", + excluded + ); + } + for reported in ["/var/lib/docker", "/var/lib/containers"] { + assert!( + reached_statfs(reported), + "a filesystem mounted exactly at {} is a real volume and must be scraped", + reported + ); + } + + // The exclusions are path-boundary aware: /devices is not below /dev. + assert!( + reached_statfs("/devices"), + "/devices must not be excluded by the /dev prefix" + ); + } + + /// `node_uname_info` is an annotation metric, so its label *values* are + /// host-dependent — but the key set, the value, the unit and the type are not. + #[test] + fn test_collect_uname() { + let tmp = tempfile::tempdir().unwrap(); + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_uname(&config_for(vec![Collector::Uname])); + + assert_eq!(metrics.len(), 1); + let info = &metrics[0]; + assert_eq!(info.name, "node_uname_info"); + // An info metric carries no measurement: the labels are the payload. + assert_eq!(info.value, 1.0); + assert_eq!(info.unit, None); + assert_eq!(info.metric_type, MetricType::Gauge); + + // Order matters as well as membership: these keys are the series identity, and + // `domainname` exists only in the Linux `utsname`. + let keys: Vec<&str> = info.labels.iter().map(|(k, _)| k.as_str()).collect(); + #[cfg(target_os = "linux")] + assert_eq!( + keys, + [ + "sysname", + "nodename", + "release", + "version", + "machine", + "domainname" + ] + ); + #[cfg(not(target_os = "linux"))] + assert_eq!( + keys, + ["sysname", "nodename", "release", "version", "machine"] + ); + + // uname(2) always fills these, so an empty value would mean the fixed-size C + // character array was decoded wrongly rather than that the host is unusual. + for key in ["sysname", "release", "machine"] { + assert!( + !label(info, key).is_empty(), + "{} should be populated from uname(2)", + key + ); + } + + // A filtered-out uname metric is dropped entirely, not emitted unlabelled. + let filtered = config_for(vec![Collector::Uname]) + .with_exclude_filter("^node_uname_info$") + .unwrap(); + assert!(collector.collect_uname(&filtered).is_empty()); + } + + #[test] + fn test_collect_loadavg() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "loadavg", "0.52 0.58 0.59 2/1234 5678\n"); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_loadavg(&NodeMetricsConfig::default()); + + assert_eq!(metrics.len(), 3); + assert_eq!(find_one(&metrics, "node_load1", &[]).value, 0.52); + assert_eq!(find_one(&metrics, "node_load5", &[]).value, 0.58); + assert_eq!(find_one(&metrics, "node_load15", &[]).value, 0.59); + for metric in &metrics { + assert_eq!(metric.metric_type, MetricType::Gauge); + assert!(metric.labels.is_empty()); + assert_eq!(metric.unit, None); + } + } + + // --------------------------------------------------------------------- + // Gauge/counter classification + // --------------------------------------------------------------------- + + #[test] + fn test_vmstat_gauge_counter_classification() { + let tmp = tempfile::tempdir().unwrap(); + let vmstat_content = "\ +nr_free_pages 123456 +nr_anon_pages 654 +nr_zone_inactive_anon 7890 +nr_dirtied 11 +nr_written 12 +nr_throttled_written 13 +nr_vmscan_write 13 +nr_vmscan_immediate_reclaim 14 +nr_foll_pin_acquired 15 +nr_foll_pin_released 16 +nr_tlb_remote_flush 17 +nr_tlb_local_flush_all 18 +pgfault 999999 +pgmajfault 1234 +pswpin 100 +pswpout 200 +workingset_nodes 500 +workingset_refault_anon 300 +numa_hit 42 +oom_kill 1 +"; + std::fs::write(tmp.path().join("vmstat"), vmstat_content).unwrap(); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let config = NodeMetricsConfig::default(); + let metrics = collector.collect_vmstat(&config); + + let find = |name: &str| metrics.iter().find(|m| m.name == name); + + // nr_* fields are gauges + assert_eq!( + find("node_vmstat_nr_free_pages").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_vmstat_nr_zone_inactive_anon") + .unwrap() + .metric_type, + MetricType::Gauge + ); + + // ... except for the cumulative event counters that share the nr_ prefix. + // These are the control cases for the exception list: without it they would + // silently be exported as gauges and lose their monotonic semantics. + for (name, value) in [ + ("node_vmstat_nr_dirtied", 11.0), + ("node_vmstat_nr_written", 12.0), + ("node_vmstat_nr_throttled_written", 13.0), + ("node_vmstat_nr_vmscan_write", 13.0), + ("node_vmstat_nr_vmscan_immediate_reclaim", 14.0), + ("node_vmstat_nr_foll_pin_acquired", 15.0), + ("node_vmstat_nr_foll_pin_released", 16.0), + // Every nr_tlb_* field is a cumulative flush count. + ("node_vmstat_nr_tlb_remote_flush", 17.0), + ("node_vmstat_nr_tlb_local_flush_all", 18.0), + ] { + let metric = find(name).unwrap_or_else(|| panic!("{} not collected", name)); + assert_eq!(metric.metric_type, MetricType::Counter, "{}", name); + assert_eq!(metric.value, value, "{}", name); + } + + // A neighbouring page-count gauge, to pin that the exception list is + // name-exact rather than a blanket "nr_ is a counter" rule. + let anon = find("node_vmstat_nr_anon_pages").unwrap(); + assert_eq!(anon.metric_type, MetricType::Gauge); + assert_eq!(anon.value, 654.0); + + // vmstat fields carry no unit and no labels, whichever type they get. + for metric in &metrics { + assert_eq!(metric.unit, None, "{}", metric.name); + assert!(metric.labels.is_empty(), "{}", metric.name); + } + + // workingset_nodes is the one gauge exception + let workingset_nodes = find("node_vmstat_workingset_nodes").unwrap(); + assert_eq!(workingset_nodes.metric_type, MetricType::Gauge); + assert_eq!(workingset_nodes.value, 500.0); + + // workingset_refault_* are counters + let refault = find("node_vmstat_workingset_refault_anon").unwrap(); + assert_eq!(refault.metric_type, MetricType::Counter); + assert_eq!(refault.value, 300.0); + + // pg*, psw*, numa_*, oom_kill are counters + assert_eq!( + find("node_vmstat_pgfault").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_pgmajfault").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_pswpin").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_pswpout").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_numa_hit").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_oom_kill").unwrap().metric_type, + MetricType::Counter + ); + } + + #[test] + fn test_netstat_snmp_gauge_counter_classification() { + let tmp = tempfile::tempdir().unwrap(); + let net_dir = tmp.path().join("net"); + std::fs::create_dir(&net_dir).unwrap(); + + // /proc/net/snmp contains CurrEstab (gauge) and InSegs (counter) + let snmp_content = "\ +Tcp: RtoAlgorithm RtoMin RtoMax MaxConn ActiveOpens PassiveOpens AttemptFails EstabResets CurrEstab InSegs OutSegs RetransSegs InErrs OutRsts InCsumErrors +Tcp: 1 200 120000 -1 500 100 10 5 42 99999 88888 100 2 50 0 +Ip: Forwarding DefaultTTL InReceives InHdrErrors +Ip: 1 64 123456 0 +"; + std::fs::write(net_dir.join("snmp"), snmp_content).unwrap(); + + // /proc/net/netstat — all fields are counters + let netstat_content = "\ +TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed +TcpExt: 10 20 5 +"; + std::fs::write(net_dir.join("netstat"), netstat_content).unwrap(); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let config = NodeMetricsConfig::default(); + let metrics = collector.collect_netstat(&config); + + let find = |name: &str| metrics.iter().find(|m| m.name == name); + + // Known gauges from /proc/net/snmp + assert_eq!( + find("node_netstat_Tcp_CurrEstab").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Tcp_RtoAlgorithm").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Tcp_RtoMin").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Tcp_RtoMax").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Tcp_MaxConn").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Ip_Forwarding").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Ip_DefaultTTL").unwrap().metric_type, + MetricType::Gauge + ); + + // Counters from /proc/net/snmp + assert_eq!( + find("node_netstat_Tcp_ActiveOpens").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_netstat_Tcp_InSegs").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_netstat_Ip_InReceives").unwrap().metric_type, + MetricType::Counter + ); + + // Counters from /proc/net/netstat + assert_eq!( + find("node_netstat_TcpExt_SyncookiesSent") + .unwrap() + .metric_type, + MetricType::Counter + ); + } + + #[test] + fn test_netstat_resynchronises_after_an_unpaired_line() { + let tmp = tempfile::tempdir().unwrap(); + + // The IpExt header has no value line: the following line is the next + // protocol's header. Advancing by two would consume MPTcpExt's header as if + // it were a value line and lose every protocol after the anomaly. + write( + tmp.path(), + "net/netstat", + "\ +TcpExt: SyncookiesSent SyncookiesRecv +TcpExt: 10 20 +IpExt: InNoRoutes InTruncatedPkts +MPTcpExt: MPCapableSYNRX MPCapableACKRX +MPTcpExt: 7 8 +", + ); + + // Another shape of anomaly: a stray line the parser cannot pair with + // anything, sitting between two well-formed protocol blocks. A blank line + // follows, as a kernel with no MPTcpExt support leaves behind. + write( + tmp.path(), + "net/snmp", + "\ +Ip: Forwarding DefaultTTL InReceives +Ip: 1 64 123456 +some unexpected line the kernel never wrote +Udp: InDatagrams NoPorts +Udp: 100 5 + +Tcp: CurrEstab InSegs +Tcp: 42 99999 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_netstat(&NodeMetricsConfig::default()); + + // Everything before the anomaly is unaffected. + assert_eq!( + find_one(&metrics, "node_netstat_TcpExt_SyncookiesSent", &[]).value, + 10.0 + ); + assert_eq!( + find_one(&metrics, "node_netstat_TcpExt_SyncookiesRecv", &[]).value, + 20.0 + ); + + // The unpaired header contributes nothing, rather than pairing up with the + // next protocol's header and mislabelling its keys. + assert!(matching(&metrics, "node_netstat_IpExt_InNoRoutes", &[]).is_empty()); + assert!(matching(&metrics, "node_netstat_IpExt_InTruncatedPkts", &[]).is_empty()); + + // The protocol *after* the anomaly is still collected, with the right value: + // the parser resynchronised on the next header instead of staying one line + // out of step for the rest of the file. + let mptcp = find_one(&metrics, "node_netstat_MPTcpExt_MPCapableSYNRX", &[]); + assert_eq!(mptcp.value, 7.0); + assert_eq!(mptcp.metric_type, MetricType::Counter); + assert!(mptcp.labels.is_empty()); + assert_eq!(mptcp.unit, None); + assert_eq!( + find_one(&metrics, "node_netstat_MPTcpExt_MPCapableACKRX", &[]).value, + 8.0 + ); + + // Same in /proc/net/snmp: the block after the stray line survives, and the + // gauge/counter classification is still applied to the recovered keys. + assert_eq!( + find_one(&metrics, "node_netstat_Ip_InReceives", &[]).value, + 123_456.0 + ); + let udp = find_one(&metrics, "node_netstat_Udp_InDatagrams", &[]); + assert_eq!(udp.value, 100.0); + assert_eq!(udp.metric_type, MetricType::Counter); + assert_eq!( + find_one(&metrics, "node_netstat_Udp_NoPorts", &[]).value, + 5.0 + ); + + let curr_estab = find_one(&metrics, "node_netstat_Tcp_CurrEstab", &[]); + assert_eq!(curr_estab.value, 42.0); + assert_eq!(curr_estab.metric_type, MetricType::Gauge); + assert_eq!( + find_one(&metrics, "node_netstat_Tcp_InSegs", &[]).value, + 99_999.0 + ); + } +} diff --git a/src/receivers/node_metrics/collector/sysfs.rs b/src/receivers/node_metrics/collector/sysfs.rs new file mode 100644 index 00000000..dea86aea --- /dev/null +++ b/src/receivers/node_metrics/collector/sysfs.rs @@ -0,0 +1,991 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! sysfs-backed collectors: CPU frequency, thermal zones, NVMe and hwmon sensors. +//! +//! Metric names, HELP strings and the hwmon chip-naming approach are derived from +//! Prometheus node_exporter — see the notice in this module's `mod.rs`. + +use crate::receivers::node_metrics::collector::{CollectedMetric, MetricType, SystemCollector}; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use std::fs; +use tracing::debug; + +impl SystemCollector { + /// Collect CPU frequency metrics from /sys/devices/system/cpu/cpu*/cpufreq/ + pub(super) fn collect_cpufreq(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + let cpu_dir = format!("{}/devices/system/cpu", self.sysfs_path); + + let entries = match fs::read_dir(&cpu_dir) { + Ok(e) => e, + Err(e) => { + debug!("Failed to read {}: {}", cpu_dir, e); + return metrics; + } + }; + + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + // Match cpu0, cpu1, etc. + let cpu_num = match name_str.strip_prefix("cpu") { + Some(n) if n.chars().all(|c| c.is_ascii_digit()) && !n.is_empty() => n.to_string(), + _ => continue, + }; + + let cpufreq_dir = entry.path().join("cpufreq"); + + let freq_files = [ + ( + "scaling_cur_freq", + "node_cpu_scaling_frequency_hertz", + "Current scaled CPU frequency", + ), + ( + "scaling_min_freq", + "node_cpu_scaling_frequency_min_hertz", + "Minimum scaled CPU frequency", + ), + ( + "scaling_max_freq", + "node_cpu_scaling_frequency_max_hertz", + "Maximum scaled CPU frequency", + ), + ( + "cpuinfo_cur_freq", + "node_cpu_frequency_hertz", + "Current CPU frequency from hardware", + ), + ( + "cpuinfo_min_freq", + "node_cpu_frequency_min_hertz", + "Minimum CPU frequency from hardware", + ), + ( + "cpuinfo_max_freq", + "node_cpu_frequency_max_hertz", + "Maximum CPU frequency from hardware", + ), + ]; + + for (file, metric_name, description) in &freq_files { + if !config.should_include_metric(metric_name) { + continue; + } + + let path = cpufreq_dir.join(file); + if let Ok(content) = fs::read_to_string(&path) { + // Values in cpufreq are in kHz, convert to Hz + if let Ok(khz) = content.trim().parse::() { + metrics.push(CollectedMetric { + name: metric_name.to_string(), + value: khz * 1000.0, + labels: vec![("cpu".to_string(), cpu_num.clone())], + metric_type: MetricType::Gauge, + unit: Some("Hz".to_string()), + description: Some(description.to_string()), + }); + } + } + } + } + + metrics + } + + /// Collect thermal zone temperatures from /sys/class/thermal/thermal_zone*/ + pub(super) fn collect_thermal_zone(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + let thermal_dir = format!("{}/class/thermal", self.sysfs_path); + + let entries = match fs::read_dir(&thermal_dir) { + Ok(e) => e, + Err(e) => { + debug!("Failed to read {}: {}", thermal_dir, e); + return metrics; + } + }; + + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + let zone_num = match name_str.strip_prefix("thermal_zone") { + Some(n) if n.chars().all(|c| c.is_ascii_digit()) && !n.is_empty() => n.to_string(), + _ => continue, + }; + + let zone_path = entry.path(); + + // Read zone type (e.g., "x86_pkg_temp", "acpitz") + let zone_type = fs::read_to_string(zone_path.join("type")) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + + // Read temperature (millidegrees Celsius -> Celsius) + if config.should_include_metric("node_thermal_zone_temp") + && let Ok(content) = fs::read_to_string(zone_path.join("temp")) + && let Ok(millideg) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_thermal_zone_temp".to_string(), + value: millideg / 1000.0, + labels: vec![ + ("zone".to_string(), zone_num.clone()), + ("type".to_string(), zone_type.clone()), + ], + metric_type: MetricType::Gauge, + unit: Some("Cel".to_string()), + description: Some("Zone temperature in Celsius".to_string()), + }); + } + } + + // Cooling devices + if (config.should_include_metric("node_cooling_device_cur_state") + || config.should_include_metric("node_cooling_device_max_state")) + && let Ok(entries) = fs::read_dir(&thermal_dir) + { + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + // Require the trailing index, like the cpu/thermal_zone/nvme scans do + let is_cooling_device = name_str + .strip_prefix("cooling_device") + .is_some_and(|r| !r.is_empty() && r.chars().all(|c| c.is_ascii_digit())); + if !is_cooling_device { + continue; + } + + let dev_path = entry.path(); + // Use bare index ("0", "1") matching Prometheus node_exporter convention. + // The guard above already required the prefix, so this cannot be empty. + let dev_name = name_str + .strip_prefix("cooling_device") + .unwrap_or_default() + .to_string(); + + let dev_type = fs::read_to_string(dev_path.join("type")) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + + let labels = vec![ + ("name".to_string(), dev_name), + ("type".to_string(), dev_type), + ]; + + for (file, metric_name, desc) in [ + ( + "cur_state", + "node_cooling_device_cur_state", + "Current state of the cooling device", + ), + ( + "max_state", + "node_cooling_device_max_state", + "Maximum state of the cooling device", + ), + ] { + if !config.should_include_metric(metric_name) { + continue; + } + if let Ok(content) = fs::read_to_string(dev_path.join(file)) + && let Ok(val) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: metric_name.to_string(), + value: val, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: None, + description: Some(desc.to_string()), + }); + } + } + } + } + + metrics + } + + /// Collect NVMe device info from /sys/class/nvme/ + pub(super) fn collect_nvme(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + if !config.should_include_metric("node_nvme_info") { + return metrics; + } + + let nvme_dir = format!("{}/class/nvme", self.sysfs_path); + let entries = match fs::read_dir(&nvme_dir) { + Ok(e) => e, + Err(e) => { + debug!("Failed to read {}: {}", nvme_dir, e); + return metrics; + } + }; + + for entry in entries.flatten() { + let dev_path = entry.path(); + let dev_name = entry.file_name().to_string_lossy().to_string(); + + // Only real controller entries (nvme0, nvme1, ...), consistent with the + // cpufreq and thermal-zone collectors. A stray file in this directory would + // otherwise produce an info metric with five empty labels. + let is_controller = dev_name + .strip_prefix("nvme") + .is_some_and(|rest| !rest.is_empty() && rest.chars().all(|c| c.is_ascii_digit())); + if !is_controller { + debug!("Skipping non-controller entry {} in {}", dev_name, nvme_dir); + continue; + } + + let read_attr = |attr: &str| -> String { + fs::read_to_string(dev_path.join(attr)) + .map(|s| s.trim().to_string()) + .unwrap_or_default() + }; + + metrics.push(CollectedMetric { + name: "node_nvme_info".to_string(), + value: 1.0, + labels: vec![ + ("device".to_string(), dev_name), + ("firmware_revision".to_string(), read_attr("firmware_rev")), + ("model".to_string(), read_attr("model")), + ("serial".to_string(), read_attr("serial")), + ("state".to_string(), read_attr("state")), + ], + metric_type: MetricType::Gauge, + unit: None, + description: Some("NVMe device information".to_string()), + }); + } + + metrics + } + + /// Collect hardware monitoring sensor data from /sys/class/hwmon/ + pub(super) fn collect_hwmon(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + let hwmon_dir = format!("{}/class/hwmon", self.sysfs_path); + + let entries = match fs::read_dir(&hwmon_dir) { + Ok(e) => e, + Err(e) => { + debug!("Failed to read {}: {}", hwmon_dir, e); + return metrics; + } + }; + + for entry in entries.flatten() { + let hwmon_path = entry.path(); + let hwmon_dir_name = entry.file_name().to_string_lossy().to_string(); + + // Require the trailing index, like the cpu/thermal_zone/nvme/cooling_device + // scans do, so a stray file in this directory is not treated as a chip. + let is_hwmon_dir = hwmon_dir_name + .strip_prefix("hwmon") + .is_some_and(|rest| !rest.is_empty() && rest.chars().all(|c| c.is_ascii_digit())); + if !is_hwmon_dir { + debug!( + "Skipping non-chip entry {} in {}", + hwmon_dir_name, hwmon_dir + ); + continue; + } + + // The human-readable model name, published separately as an annotation metric. + // It is not usable as an identity on its own: it is a chip *model* and repeats + // across instances (two NVMe drives both report "nvme"). + let chip_name = fs::read_to_string(hwmon_path.join("name")) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + + // The `chip` label must uniquely identify the hwmon instance, otherwise two + // chips of the same model produce identical attribute sets and one of them is + // dropped. The backing device name is both unique and stable, so prefer it. + // Without a `device` symlink, fall back to an identity that still includes the + // hwmonN directory, which is unique within a host — the model name alone is + // not. That index depends on module load order, so a fallback identity is not + // stable across reboots. + let chip = fs::read_link(hwmon_path.join("device")) + .ok() + .and_then(|target| { + target + .file_name() + .map(|n| sanitize_hwmon_chip(&n.to_string_lossy())) + }) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| match &chip_name { + Some(name) => format!("{}_{}", sanitize_hwmon_chip(name), hwmon_dir_name), + None => hwmon_dir_name.clone(), + }); + + if let Some(chip_name) = &chip_name + && config.should_include_metric("node_hwmon_chip_names") + { + metrics.push(CollectedMetric { + name: "node_hwmon_chip_names".to_string(), + value: 1.0, + labels: vec![ + ("chip".to_string(), chip.clone()), + ("chip_name".to_string(), chip_name.clone()), + ], + metric_type: MetricType::Gauge, + unit: None, + description: Some( + "Annotation metric for human-readable chip names".to_string(), + ), + }); + } + + // Scan for sensor files: temp*, in*, fan*, power* and curr* `_input` readings + let dir_entries = match fs::read_dir(&hwmon_path) { + Ok(e) => e, + Err(_) => continue, + }; + + for sensor_entry in dir_entries.flatten() { + let fname = sensor_entry.file_name(); + let fname_str = fname.to_string_lossy(); + + // Only process *_input files — these are the primary readings + let Some(sensor_name) = fname_str.strip_suffix("_input") else { + continue; + }; + + // hwmon sensor files are named _input. Match on the alphabetic + // prefix exactly rather than with `starts_with`, so that e.g. `intrusion0` + // is not mistaken for a voltage input (`in`). + let kind: &str = &sensor_name[..sensor_name + .find(|c: char| !c.is_ascii_alphabetic()) + .unwrap_or(sensor_name.len())]; + + let (metric_name, divisor, unit, desc) = match kind { + // temp1_input, temp2_input, etc. — millidegrees C + "temp" => ( + "node_hwmon_temp_celsius", + 1000.0, + "Cel", + "Hardware monitor temperature", + ), + // in0_input, in1_input, etc. — millivolts + "in" => ( + "node_hwmon_in_volts", + 1000.0, + "V", + "Hardware monitor voltage", + ), + // fan1_input, fan2_input, etc. — RPM (no conversion) + "fan" => ( + "node_hwmon_fan_rpm", + 1.0, + "{rev}/min", + "Hardware monitor fan speed", + ), + // power1_input, etc. — microwatts + "power" => ( + "node_hwmon_power_watts", + 1_000_000.0, + "W", + "Hardware monitor power consumption", + ), + // curr1_input, etc. — milliamps + "curr" => ( + "node_hwmon_curr_amps", + 1000.0, + "A", + "Hardware monitor current", + ), + _ => { + debug!( + "Skipping unsupported hwmon sensor {} on chip {}", + sensor_name, chip + ); + continue; + } + }; + + // The threshold metrics below check their own names, so this sensor is only + // skipped entirely when none of its metrics are wanted. Gating everything + // on `metric_name` would make the thresholds unreachable whenever the + // reading itself is filtered out. + let want_reading = config.should_include_metric(metric_name); + let want_thresholds = kind == "temp" + && (config.should_include_metric("node_hwmon_temp_max_celsius") + || config.should_include_metric("node_hwmon_temp_crit_celsius")); + if !want_reading && !want_thresholds { + continue; + } + + // The `sensor` label is the raw sysfs name (`temp1`, `in0`), which is unique + // within a chip, matching Prometheus node_exporter. The human-readable text + // from `_label` is published separately as an annotation metric: + // using it as the identity would collide whenever two sensors on one chip + // carry the same text, and would move series whenever a driver gains or + // changes a label file. + let labels = vec![ + ("chip".to_string(), chip.clone()), + ("sensor".to_string(), sensor_name.to_string()), + ]; + + let label_path = hwmon_path.join(format!("{}_label", sensor_name)); + if let Some(sensor_label) = fs::read_to_string(&label_path) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + && config.should_include_metric("node_hwmon_sensor_label") + { + metrics.push(CollectedMetric { + name: "node_hwmon_sensor_label".to_string(), + value: 1.0, + labels: vec![ + ("chip".to_string(), chip.clone()), + ("sensor".to_string(), sensor_name.to_string()), + ("label".to_string(), sensor_label), + ], + metric_type: MetricType::Gauge, + unit: None, + description: Some( + "Annotation metric for human-readable sensor labels".to_string(), + ), + }); + } + + if want_reading + && let Ok(content) = fs::read_to_string(sensor_entry.path()) + && let Ok(raw) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: metric_name.to_string(), + value: raw / divisor, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: Some(unit.to_string()), + description: Some(desc.to_string()), + }); + } + + // For temperature sensors, also read max and crit thresholds + if kind == "temp" { + let max_path = hwmon_path.join(format!("{}_max", sensor_name)); + if config.should_include_metric("node_hwmon_temp_max_celsius") + && let Ok(content) = fs::read_to_string(&max_path) + && let Ok(raw) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_hwmon_temp_max_celsius".to_string(), + value: raw / 1000.0, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: Some("Cel".to_string()), + description: Some( + "Hardware monitor temperature max threshold".to_string(), + ), + }); + } + + let crit_path = hwmon_path.join(format!("{}_crit", sensor_name)); + if config.should_include_metric("node_hwmon_temp_crit_celsius") + && let Ok(content) = fs::read_to_string(&crit_path) + && let Ok(raw) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_hwmon_temp_crit_celsius".to_string(), + value: raw / 1000.0, + labels, + metric_type: MetricType::Gauge, + unit: Some("Cel".to_string()), + description: Some( + "Hardware monitor temperature critical threshold".to_string(), + ), + }); + } + } + } + } + + metrics + } +} + +/// Make an hwmon chip identity safe to use as a metric label value. +/// +/// Non-alphanumeric characters are replaced with `_`, following the same approach as +/// Prometheus `node_exporter`. This is a lossy mapping (`coretemp.0` and `coretemp-0` +/// both become `coretemp_0`), so it does not by itself guarantee uniqueness — a backing +/// device name is unique on its own, and the fallback identities compose the hwmon +/// directory to stay so. Values also differ from node_exporter's, which prefixes the +/// device's parent directory (`platform_coretemp_0` where this produces `coretemp_0`). +fn sanitize_hwmon_chip(device_name: &str) -> String { + device_name + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::receivers::node_metrics::collector::test_support::*; + use crate::receivers::node_metrics::config::Collector; + use std::os::unix::fs::symlink; + + #[test] + fn test_collect_cpufreq() { + let sys = tempfile::tempdir().unwrap(); + write( + sys.path(), + "devices/system/cpu/cpu0/cpufreq/scaling_cur_freq", + "2400000\n", + ); + write( + sys.path(), + "devices/system/cpu/cpu0/cpufreq/scaling_min_freq", + "800000\n", + ); + write( + sys.path(), + "devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq", + "3600000\n", + ); + write( + sys.path(), + "devices/system/cpu/cpu1/cpufreq/scaling_cur_freq", + "1200000\n", + ); + // Siblings of the cpu directories that must not be scanned as CPUs. + write(sys.path(), "devices/system/cpu/cpufreq/boost", "1\n"); + write( + sys.path(), + "devices/system/cpu/cpuidle/current_driver", + "none\n", + ); + write(sys.path(), "devices/system/cpu/cpu_capacity", "1024\n"); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_cpufreq(&config_for(vec![Collector::Cpufreq])); + + assert_eq!(label_values(&metrics, "cpu"), ["0", "1"]); + assert_eq!(metrics.len(), 4); + + // cpufreq reports kHz; the metric is Hz. + let cur = find_one( + &metrics, + "node_cpu_scaling_frequency_hertz", + &[("cpu", "0")], + ); + assert_eq!(cur.value, 2_400_000_000.0); + assert_eq!(cur.unit.as_deref(), Some("Hz")); + assert_eq!(cur.metric_type, MetricType::Gauge); + + assert_eq!( + find_one( + &metrics, + "node_cpu_scaling_frequency_min_hertz", + &[("cpu", "0")] + ) + .value, + 800_000_000.0 + ); + assert_eq!( + find_one(&metrics, "node_cpu_frequency_max_hertz", &[("cpu", "0")]).value, + 3_600_000_000.0 + ); + assert_eq!( + find_one( + &metrics, + "node_cpu_scaling_frequency_hertz", + &[("cpu", "1")] + ) + .value, + 1_200_000_000.0 + ); + } + + #[test] + fn test_collect_thermal_zone() { + let sys = tempfile::tempdir().unwrap(); + write( + sys.path(), + "class/thermal/thermal_zone0/type", + "x86_pkg_temp\n", + ); + write(sys.path(), "class/thermal/thermal_zone0/temp", "45000\n"); + write( + sys.path(), + "class/thermal/cooling_device0/type", + "Processor\n", + ); + write(sys.path(), "class/thermal/cooling_device0/cur_state", "0\n"); + write(sys.path(), "class/thermal/cooling_device0/max_state", "3\n"); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_thermal_zone(&config_for(vec![Collector::ThermalZone])); + + // Millidegrees Celsius in the file, degrees Celsius in the metric. + let temp = find_one( + &metrics, + "node_thermal_zone_temp", + &[("zone", "0"), ("type", "x86_pkg_temp")], + ); + assert_eq!(temp.value, 45.0); + assert_eq!(temp.unit.as_deref(), Some("Cel")); + assert_eq!(temp.metric_type, MetricType::Gauge); + + let cooling = [("name", "0"), ("type", "Processor")]; + assert_eq!( + find_one(&metrics, "node_cooling_device_cur_state", &cooling).value, + 0.0 + ); + assert_eq!( + find_one(&metrics, "node_cooling_device_max_state", &cooling).value, + 3.0 + ); + assert_eq!(metrics.len(), 3); + } + + #[test] + fn test_collect_nvme() { + let sys = tempfile::tempdir().unwrap(); + write( + sys.path(), + "class/nvme/nvme0/model", + "Samsung SSD 980 PRO 1TB\n", + ); + write(sys.path(), "class/nvme/nvme0/serial", "S5GXNX0R123456\n"); + write(sys.path(), "class/nvme/nvme0/firmware_rev", "5B2QGXA7\n"); + write(sys.path(), "class/nvme/nvme0/state", "live\n"); + // Entries that are not controllers: a namespace directory, and a plain file. + // Both carry none of the controller attributes, so accepting them would emit + // an info metric whose five labels are all empty. + write(sys.path(), "class/nvme/nvme0n1/size", "1953525168\n"); + write(sys.path(), "class/nvme/uevent", "\n"); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_nvme(&config_for(vec![Collector::Nvme])); + + assert_eq!(metrics.len(), 1); + assert_eq!(label_values(&metrics, "device"), ["nvme0"]); + let info = &metrics[0]; + assert_eq!(info.name, "node_nvme_info"); + assert_eq!(info.value, 1.0); + assert_eq!(info.metric_type, MetricType::Gauge); + assert_eq!(info.unit, None); + assert_eq!(label(info, "device"), "nvme0"); + assert_eq!(label(info, "model"), "Samsung SSD 980 PRO 1TB"); + assert_eq!(label(info, "serial"), "S5GXNX0R123456"); + assert_eq!(label(info, "firmware_revision"), "5B2QGXA7"); + assert_eq!(label(info, "state"), "live"); + + // The label set is exactly these five keys, so a stray entry cannot hide as + // an extra series with the same name. + let mut keys: Vec<&str> = info.labels.iter().map(|(k, _)| k.as_str()).collect(); + keys.sort_unstable(); + assert_eq!( + keys, + ["device", "firmware_revision", "model", "serial", "state"] + ); + } + + #[test] + fn test_collect_hwmon() { + let sys = tempfile::tempdir().unwrap(); + write(sys.path(), "class/hwmon/hwmon0/name", "coretemp\n"); + write(sys.path(), "class/hwmon/hwmon0/temp1_input", "42000\n"); + write( + sys.path(), + "class/hwmon/hwmon0/temp1_label", + "Package id 0\n", + ); + write(sys.path(), "class/hwmon/hwmon0/temp1_max", "84000\n"); + write(sys.path(), "class/hwmon/hwmon0/temp1_crit", "100000\n"); + // No label file at all. + write(sys.path(), "class/hwmon/hwmon0/temp2_input", "41000\n"); + // Present but empty label file. + write(sys.path(), "class/hwmon/hwmon0/temp3_input", "40000\n"); + write(sys.path(), "class/hwmon/hwmon0/temp3_label", "\n"); + write(sys.path(), "class/hwmon/hwmon0/in0_input", "1200\n"); + write(sys.path(), "class/hwmon/hwmon0/fan1_input", "1500\n"); + write(sys.path(), "class/hwmon/hwmon0/power1_input", "7500000\n"); + write(sys.path(), "class/hwmon/hwmon0/curr1_input", "850\n"); + // Not a voltage input, despite sharing the "in" prefix. + write(sys.path(), "class/hwmon/hwmon0/intrusion0_input", "0\n"); + symlink( + "../../devices/platform/coretemp.0", + sys.path().join("class/hwmon/hwmon0/device"), + ) + .unwrap(); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_hwmon(&config_for(vec![Collector::Hwmon])); + + // The chip identity comes from the backing device, sanitised for use as a label. + let chip = "coretemp_0"; + assert_eq!(label_values(&metrics, "chip"), [chip]); + + let names = find_one( + &metrics, + "node_hwmon_chip_names", + &[("chip", chip), ("chip_name", "coretemp")], + ); + assert_eq!(names.value, 1.0); + // Info-style annotation metrics carry no unit, like node_nvme_info/node_uname_info + assert_eq!(names.unit, None); + + // Temperatures, voltages and currents are milli-units; power is micro; + // fan speed is already RPM. + let temp1 = find_one( + &metrics, + "node_hwmon_temp_celsius", + &[("chip", chip), ("sensor", "temp1")], + ); + assert_eq!(temp1.value, 42.0); + assert_eq!(temp1.unit.as_deref(), Some("Cel")); + assert_eq!(temp1.metric_type, MetricType::Gauge); + + assert_eq!( + find_one( + &metrics, + "node_hwmon_temp_max_celsius", + &[("chip", chip), ("sensor", "temp1")] + ) + .value, + 84.0 + ); + assert_eq!( + find_one( + &metrics, + "node_hwmon_temp_crit_celsius", + &[("chip", chip), ("sensor", "temp1")] + ) + .value, + 100.0 + ); + + let volts = find_one(&metrics, "node_hwmon_in_volts", &[("sensor", "in0")]); + assert_eq!(volts.value, 1.2); + assert_eq!(volts.unit.as_deref(), Some("V")); + + let fan = find_one(&metrics, "node_hwmon_fan_rpm", &[("sensor", "fan1")]); + assert_eq!(fan.value, 1500.0); + assert_eq!(fan.unit.as_deref(), Some("{rev}/min")); + + let power = find_one(&metrics, "node_hwmon_power_watts", &[("sensor", "power1")]); + assert_eq!(power.value, 7.5); + assert_eq!(power.unit.as_deref(), Some("W")); + + let curr = find_one(&metrics, "node_hwmon_curr_amps", &[("sensor", "curr1")]); + assert_eq!(curr.value, 0.85); + assert_eq!(curr.unit.as_deref(), Some("A")); + + // Sensors are identified by their raw sysfs name whether or not they carry a label + // file, so two sensors on one chip can never collide. + assert_eq!( + find_one(&metrics, "node_hwmon_temp_celsius", &[("sensor", "temp2")]).value, + 41.0 + ); + assert_eq!( + find_one(&metrics, "node_hwmon_temp_celsius", &[("sensor", "temp3")]).value, + 40.0 + ); + assert!(matching(&metrics, "node_hwmon_temp_celsius", &[("sensor", "")]).is_empty()); + + // The human-readable text is published separately, keyed by chip and sensor, as + // Prometheus node_exporter does. Only the sensor that has a non-empty label file + // gets one. + let sensor_label = find_one( + &metrics, + "node_hwmon_sensor_label", + &[ + ("chip", chip), + ("sensor", "temp1"), + ("label", "Package id 0"), + ], + ); + assert_eq!(sensor_label.value, 1.0); + assert_eq!(sensor_label.unit, None); + // ...and only that one sensor gets an annotation + let labelled: Vec = matching(&metrics, "node_hwmon_sensor_label", &[]) + .iter() + .map(|m| label(m, "sensor").to_string()) + .collect(); + assert_eq!(labelled, ["temp1"]); + + // intrusion0_input is not a supported sensor kind and is skipped. + assert_eq!(matching(&metrics, "node_hwmon_in_volts", &[]).len(), 1); + } + + #[test] + fn test_collect_hwmon_same_model_chips_get_distinct_labels() { + let sys = tempfile::tempdir().unwrap(); + for (hwmon, device) in [("hwmon1", "nvme0"), ("hwmon2", "nvme1")] { + write(sys.path(), &format!("class/hwmon/{}/name", hwmon), "nvme\n"); + write( + sys.path(), + &format!("class/hwmon/{}/temp1_input", hwmon), + "38000\n", + ); + symlink( + format!("../../devices/pci0000:00/0000:00:1d.0/nvme/{}", device), + sys.path().join(format!("class/hwmon/{}/device", hwmon)), + ) + .unwrap(); + } + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_hwmon(&config_for(vec![Collector::Hwmon])); + + // Two chips of the same model must not collapse onto one attribute set: + // the `chip` label identifies the instance, `chip_name` the model. + assert_eq!(label_values(&metrics, "chip"), ["nvme0", "nvme1"]); + assert_eq!( + matching(&metrics, "node_hwmon_temp_celsius", &[("sensor", "temp1")]).len(), + 2 + ); + for chip in ["nvme0", "nvme1"] { + assert_eq!( + find_one( + &metrics, + "node_hwmon_temp_celsius", + &[("chip", chip), ("sensor", "temp1")] + ) + .value, + 38.0 + ); + find_one( + &metrics, + "node_hwmon_chip_names", + &[("chip", chip), ("chip_name", "nvme")], + ); + } + } + + /// The three ways a `chip` identity can be derived, in preference order. + /// + /// Only the first is exercised elsewhere; the fallbacks are what a host without + /// `device` symlinks (embedded platform drivers) actually takes. + /// The threshold metrics check their own names, so excluding the base reading must not + /// make them unreachable — an operator trimming cardinality still wants the limits. + #[test] + fn test_collect_hwmon_thresholds_survive_excluding_the_reading() { + let sys = tempfile::tempdir().unwrap(); + let hwmon = sys.path().join("class/hwmon/hwmon0"); + std::fs::create_dir_all(&hwmon).unwrap(); + std::fs::write(hwmon.join("name"), "coretemp\n").unwrap(); + std::fs::write(hwmon.join("temp1_input"), "42000\n").unwrap(); + std::fs::write(hwmon.join("temp1_max"), "84000\n").unwrap(); + std::fs::write(hwmon.join("temp1_crit"), "100000\n").unwrap(); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let config = NodeMetricsConfig { + collectors: vec![Collector::Hwmon], + sysfs_path: sys.path().to_str().unwrap().to_string(), + ..Default::default() + } + .with_exclude_filter("^node_hwmon_temp_celsius$") + .unwrap(); + + let metrics = collector.collect_hwmon(&config); + + assert!(matching(&metrics, "node_hwmon_temp_celsius", &[]).is_empty()); + assert_eq!( + find_one(&metrics, "node_hwmon_temp_max_celsius", &[]).value, + 84.0 + ); + assert_eq!( + find_one(&metrics, "node_hwmon_temp_crit_celsius", &[]).value, + 100.0 + ); + } + + #[test] + fn test_collect_hwmon_chip_identity_fallback_chain() { + let sys = tempfile::tempdir().unwrap(); + + // hwmon0: a `device` symlink, so the chip is the backing device name. + write(sys.path(), "class/hwmon/hwmon0/name", "coretemp\n"); + write(sys.path(), "class/hwmon/hwmon0/temp1_input", "42000\n"); + symlink( + "../../devices/platform/coretemp.0", + sys.path().join("class/hwmon/hwmon0/device"), + ) + .unwrap(); + + // hwmon1: no symlink, but a name file, so the chip combines the two. + write(sys.path(), "class/hwmon/hwmon1/name", "bq27xxx-battery\n"); + write(sys.path(), "class/hwmon/hwmon1/temp1_input", "38000\n"); + + // hwmon2: neither, so only the directory name is left to identify it. + write(sys.path(), "class/hwmon/hwmon2/temp1_input", "39000\n"); + + // hwmon3: the same model name as hwmon1 and no symlink either — the case that + // a name-only identity would collapse onto one attribute set. + write(sys.path(), "class/hwmon/hwmon3/name", "bq27xxx-battery\n"); + write(sys.path(), "class/hwmon/hwmon3/temp1_input", "40000\n"); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_hwmon(&config_for(vec![Collector::Hwmon])); + + // Four chips, four distinct identities: the device name where there is one, the + // sanitised model name plus the hwmonN directory where there is a name file, and + // the bare directory name where there is neither. + assert_eq!( + label_values(&metrics, "chip"), + [ + "bq27xxx_battery_hwmon1", + "bq27xxx_battery_hwmon3", + "coretemp_0", + "hwmon2" + ] + ); + + for (chip, temp) in [ + ("coretemp_0", 42.0), + ("bq27xxx_battery_hwmon1", 38.0), + ("hwmon2", 39.0), + ("bq27xxx_battery_hwmon3", 40.0), + ] { + let reading = find_one( + &metrics, + "node_hwmon_temp_celsius", + &[("chip", chip), ("sensor", "temp1")], + ); + assert_eq!(reading.value, temp, "chip {}", chip); + assert_eq!(reading.unit.as_deref(), Some("Cel")); + assert_eq!(reading.metric_type, MetricType::Gauge); + } + + // The invariant that matters: two chips reporting the same model with no + // `device` symlink must still be distinguishable. Identical `chip` labels would + // make them one duplicate attribute set, and one chip's readings would be + // dropped in the OTLP conversion. + let same_model: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "node_hwmon_chip_names") + .filter(|m| label(m, "chip_name") == "bq27xxx-battery") + .map(|m| label(m, "chip")) + .collect(); + assert_eq!(same_model.len(), 2, "expected both same-model chips"); + assert_ne!( + same_model[0], same_model[1], + "two chips of the same model must not share a chip label" + ); + + // The model name is only published where one exists; hwmon2 has no name file, + // so it gets no annotation metric — but its readings are still reported above. + let mut chip_names: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "node_hwmon_chip_names") + .map(|m| label(m, "chip_name")) + .collect(); + chip_names.sort_unstable(); + chip_names.dedup(); + assert_eq!(chip_names, ["bq27xxx-battery", "coretemp"]); + assert!(matching(&metrics, "node_hwmon_chip_names", &[("chip", "hwmon2")]).is_empty()); + } +} diff --git a/src/receivers/node_metrics/collector/test_support.rs b/src/receivers/node_metrics/collector/test_support.rs new file mode 100644 index 00000000..bbffb581 --- /dev/null +++ b/src/receivers/node_metrics/collector/test_support.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Fixture helpers shared by the collector unit tests. + +use crate::receivers::node_metrics::collector::CollectedMetric; +use crate::receivers::node_metrics::collector::SystemCollector; +use crate::receivers::node_metrics::config::Collector; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use std::path::Path; +use tempfile::TempDir; + +// --------------------------------------------------------------------- +// Fixture helpers +// +// Every collector takes its procfs/sysfs root as a parameter, so the whole +// module is testable from a fixture directory tree on any platform. +// --------------------------------------------------------------------- + +/// Write `contents` to `dir`/`rel`, creating any missing parent directories. +pub(super) fn write(dir: &Path, rel: &str, contents: &str) { + let path = dir.join(rel); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, contents).unwrap(); +} + +/// A default config restricted to `collectors`, with no metric filters. +pub(super) fn config_for(collectors: Vec) -> NodeMetricsConfig { + NodeMetricsConfig { + collectors, + ..Default::default() + } +} + +/// A collector reading `sysfs` as its sysfs root, with its procfs root pointed at an +/// empty fixture directory. +/// +/// For collectors that read sysfs only: passing the real `/proc` would make the test +/// depend on the host, and would read `/proc/stat` on every construction. +/// The returned [`TempDir`] must be kept alive for as long as the collector is used. +pub(super) fn collector_for_sysfs(sysfs: &Path) -> (TempDir, SystemCollector) { + let procfs = tempfile::tempdir().unwrap(); + let collector = SystemCollector::new(procfs.path().to_str().unwrap(), sysfs.to_str().unwrap()); + (procfs, collector) +} + +/// A collector whose procfs and sysfs roots are both an empty fixture directory. +/// +/// For collectors that read neither root — the textfile collector takes its directory +/// from the config — so the test touches nothing outside its own fixtures. +/// The returned [`TempDir`] must be kept alive for as long as the collector is used. +pub(super) fn collector_with_empty_roots() -> (TempDir, SystemCollector) { + let roots = tempfile::tempdir().unwrap(); + let path = roots.path().to_str().unwrap().to_string(); + let collector = SystemCollector::new(&path, &path); + (roots, collector) +} + +/// All metrics named `name` that carry every one of `labels`. +pub(super) fn matching<'a>( + metrics: &'a [CollectedMetric], + name: &str, + labels: &[(&str, &str)], +) -> Vec<&'a CollectedMetric> { + metrics + .iter() + .filter(|m| { + m.name == name + && labels + .iter() + .all(|(k, v)| m.labels.iter().any(|(mk, mv)| mk == k && mv == v)) + }) + .collect() +} + +/// The single metric named `name` carrying `labels`; panics otherwise. +pub(super) fn find_one<'a>( + metrics: &'a [CollectedMetric], + name: &str, + labels: &[(&str, &str)], +) -> &'a CollectedMetric { + let mut found = matching(metrics, name, labels); + assert_eq!( + found.len(), + 1, + "expected exactly one {} with labels {:?}, found {}", + name, + labels, + found.len() + ); + found.pop().unwrap() +} + +/// Value of label `key`, or `""` when the label is absent. +pub(super) fn label<'a>(metric: &'a CollectedMetric, key: &str) -> &'a str { + metric + .labels + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) + .unwrap_or_default() +} + +/// Sorted, deduplicated values of label `key` across `metrics`. +pub(super) fn label_values(metrics: &[CollectedMetric], key: &str) -> Vec { + let mut values: Vec = metrics.iter().map(|m| label(m, key).to_string()).collect(); + values.sort(); + values.dedup(); + values +} + +/// Total bytes the real `statfs(2)` reports for `path`. +pub(super) fn statfs_size_bytes(path: &str) -> f64 { + // Match the collector's choice of interface so the expected value is computed the same + // way on every target: Linux uses the LFS variant. + #[cfg(not(target_os = "linux"))] + use libc::{statfs, statfs as StatFs}; + #[cfg(target_os = "linux")] + use libc::{statfs64 as StatFs, statfs64 as statfs}; + + let mut stat: StatFs = unsafe { std::mem::zeroed() }; + let cpath = std::ffi::CString::new(path).unwrap(); + assert_eq!( + unsafe { statfs(cpath.as_ptr(), &mut stat) }, + 0, + "statfs({}) failed: {}", + path, + std::io::Error::last_os_error() + ); + stat.f_blocks as f64 * stat.f_bsize as f64 +} + +/// A trimmed but realistically shaped `/proc/stat`. +/// +/// `cpu2` is deliberately truncated after the `system` field, as a kernel +/// predating the later columns would report it. +pub(super) const STAT_FIXTURE: &str = "\ +cpu 3005 606 907 1200000 1500 180 210 24 2700 30 +cpu0 1000 200 300 400000 500 60 70 8 900 10 +cpu1 2000 400 600 800000 1000 120 140 16 1800 20 +cpu2 5 6 7 +intr 1234 0 0 +ctxt 987654321 +btime 1700000000 +processes 4242 +procs_running 3 +procs_blocked 1 +"; diff --git a/src/receivers/node_metrics/collector/textfile.rs b/src/receivers/node_metrics/collector/textfile.rs new file mode 100644 index 00000000..627b7f86 --- /dev/null +++ b/src/receivers/node_metrics/collector/textfile.rs @@ -0,0 +1,1223 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Textfile collector and Prometheus exposition-format parsing. +//! +//! Behaviour and the `node_textfile_*` semantics are derived from Prometheus +//! node_exporter — see the notice in this module's `mod.rs`. + +use crate::receivers::node_metrics::collector::{CollectedMetric, MetricType, SystemCollector}; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use std::collections::HashMap; +use std::fs; +use tracing::{debug, warn}; + +impl SystemCollector { + /// Collect custom metrics from Prometheus-format textfiles + /// + /// The textfile path can be either a directory (all `*.prom` files are read in + /// sorted filename order) or a single `.prom` file. + pub(super) fn collect_textfile(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let textfile_path = match &config.textfile_directory { + Some(dir) => dir, + None => { + debug!("Textfile collector enabled but no directory configured"); + return metrics; + } + }; + + let path = std::path::Path::new(textfile_path); + + // Support both single-file and directory paths + let prom_files: Vec = if path.is_file() { + vec![path.to_path_buf()] + } else { + match fs::read_dir(path) { + Ok(entries) => { + let mut files: Vec = entries + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("prom")) + // Regular files only: a FIFO or socket named `foo.prom` would + // otherwise reach `read_to_string`, where `open(2)` blocks + // indefinitely with no writer and would wedge the scrape thread. + // `unwrap_or(true)` keeps an entry whose metadata cannot be read — + // a dangling symlink, say — so it is still reported through + // `node_textfile_scrape_error` rather than disappearing. + // `fs::metadata` follows symlinks, unlike `DirEntry::file_type`, + // so a valid symlinked `.prom` is collected. + .filter(|p| fs::metadata(p).map(|m| m.is_file()).unwrap_or(true)) + .collect(); + // Read in a deterministic order: the OTLP conversion takes a metric's + // description and unit from the first sample it sees, so traversal + // order would otherwise decide which file's HELP text wins, and would + // reorder data points between scrapes. + files.sort(); + files + } + Err(e) => { + // A mistyped or unreadable directory must not fail silently, otherwise + // the receiver just emits nothing forever. + warn!( + "Failed to read textfile path {} (set via --node-metrics-receiver-textfile-directory): {}", + textfile_path, e + ); + if config.should_include_metric("node_textfile_scrape_error") { + metrics.push(textfile_scrape_error(textfile_path.clone(), 1.0)); + } + return metrics; + } + } + }; + + let mut total_samples = 0usize; + let mut over_sample_limit = false; + + for path in &prom_files { + let filename = path + .file_name() + .map(|f| f.to_string_lossy().to_string()) + .unwrap_or_default(); + + // Expose the file's modification time so a stalled writer script is + // detectable even while its (stale) metrics keep being exported. + let metadata = fs::metadata(path).ok(); + if let Some(mtime) = metadata + .as_ref() + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + && config.should_include_metric("node_textfile_mtime_seconds") + { + metrics.push(CollectedMetric { + name: "node_textfile_mtime_seconds".to_string(), + value: mtime.as_secs_f64(), + labels: vec![("filename".to_string(), filename.clone())], + metric_type: MetricType::Gauge, + unit: Some("s".to_string()), + description: Some("Modification time of the textfile in seconds".to_string()), + }); + } + + // Guard against a runaway writer script: an unbounded file would be read + // into memory in full on every scrape interval. + if let Some(size) = metadata.as_ref().map(|m| m.len()) + && size > MAX_TEXTFILE_SIZE_BYTES + { + // Logged at debug, not warn: this repeats every scrape for as long as the + // file is oversized. `node_textfile_scrape_error` is the durable signal. + debug!( + "Skipping textfile {:?}: {} bytes exceeds the {} byte limit", + path, size, MAX_TEXTFILE_SIZE_BYTES + ); + if config.should_include_metric("node_textfile_scrape_error") { + metrics.push(textfile_scrape_error(filename, 1.0)); + } + continue; + } + + let content = match read_capped(path) { + Ok(c) => c, + Err(e) => { + // Debug, like the sibling failure paths: this repeats every scrape for + // as long as the file is unreadable, and `node_textfile_scrape_error` + // is the durable signal. + debug!("Failed to read textfile {:?}: {}", path, e); + if config.should_include_metric("node_textfile_scrape_error") { + metrics.push(textfile_scrape_error(filename, 1.0)); + } + continue; + } + }; + + // Track metric types and descriptions from comments, scoped per file. + // TYPE annotations determine counter vs gauge; conflicting types for + // the same metric across files coexist and the OTLP conversion keeps + // them as separate metrics (see `convert.rs`). + let mut described: std::collections::HashSet = std::collections::HashSet::new(); + let mut metric_types: HashMap = HashMap::new(); + let mut metric_descriptions: HashMap = HashMap::new(); + let mut parsed = Vec::new(); + let mut parse_failures = 0usize; + + // Parse the file + for line in content.lines() { + let line = line.trim(); + + // Skip empty lines + if line.is_empty() { + continue; + } + + // Parse HELP comments — preserve descriptions for OTLP. + // The exposition format allows any run of whitespace after the keyword. + if let Some(rest) = strip_comment_keyword(line, "HELP") { + if let Some((name, desc)) = rest.trim_start().split_once(char::is_whitespace) + && !name.is_empty() + { + metric_descriptions + .insert(name.to_string(), unescape_exposition(desc.trim())); + } + continue; + } + + // Parse TYPE comments + if let Some(rest) = strip_comment_keyword(line, "TYPE") { + if let Some((name, type_str)) = + rest.trim_start().split_once(char::is_whitespace) + && !name.is_empty() + { + let type_str = type_str.trim(); + let metric_type = match type_str { + "counter" => MetricType::Counter, + "gauge" => MetricType::Gauge, + // histogram/summary/untyped are not modelled; their component + // series are exported as gauges. + other => { + debug!( + "Textfile {}: unsupported metric type {:?} for {}, treating as gauge", + filename, other, name + ); + MetricType::Gauge + } + }; + metric_types.insert(name.to_string(), metric_type); + } + continue; + } + + // Skip other comments + if line.starts_with('#') { + continue; + } + + // Parse metric line + match parse_prometheus_line(line, &metric_types) { + Some(mut metric) => { + if !config.should_include_metric(&metric.name) { + continue; + } + // Attach HELP description if available + // Attach the HELP text to the first sample of each metric only: the + // OTLP conversion takes the description from the first member of a + // group, so cloning it onto every sample would hold hundreds of + // thousands of copies of the same string for a large file. + if let Some(desc) = metric_descriptions.get(&metric.name) + && described.insert(metric.name.clone()) + { + metric.description = Some(desc.clone()); + } + parsed.push(metric); + } + None => { + // A malformed line must not disappear without a trace, otherwise a + // broken writer script is indistinguishable from an empty file. + parse_failures += 1; + debug!("Textfile {}: skipping malformed line {:?}", filename, line); + } + } + } + + if parse_failures > 0 { + // Debug rather than warn: a static file with a bad line would otherwise log + // on every scrape forever. `node_textfile_scrape_error` carries the signal. + debug!( + "Textfile {}: skipped {} malformed line(s)", + filename, parse_failures + ); + } + + if config.should_include_metric("node_textfile_scrape_error") { + let value = if parse_failures > 0 { 1.0 } else { 0.0 }; + metrics.push(textfile_scrape_error(filename.clone(), value)); + } + + if parsed.len() + total_samples > MAX_TEXTFILE_SAMPLES { + let room = MAX_TEXTFILE_SAMPLES.saturating_sub(total_samples); + warn!( + "Textfile {}: keeping {} of {} samples; the {} sample per-scrape limit \ + was reached", + filename, + room, + parsed.len(), + MAX_TEXTFILE_SAMPLES + ); + parsed.truncate(room); + over_sample_limit = true; + } + total_samples += parsed.len(); + metrics.append(&mut parsed); + + if over_sample_limit { + break; + } + } + + metrics + } +} + +/// Maximum number of samples taken from the textfile directory in one scrape +const MAX_TEXTFILE_SAMPLES: usize = 100_000; + +/// Read a textfile, enforcing the size cap during the read +/// +/// The pre-read `stat` is only advisory: a file can grow between the stat and the read, and +/// a non-regular file reports a length of zero. Reading through a bounded reader means the +/// cap holds regardless. +fn read_capped(path: &std::path::Path) -> std::io::Result { + use std::io::Read; + + let mut buf = String::new(); + fs::File::open(path)? + // One byte over the cap, so an oversized file is detectable rather than silently + // truncated mid-sample. + .take(MAX_TEXTFILE_SIZE_BYTES + 1) + .read_to_string(&mut buf)?; + + if buf.len() as u64 > MAX_TEXTFILE_SIZE_BYTES { + return Err(std::io::Error::other(format!( + "textfile exceeds the {} byte limit", + MAX_TEXTFILE_SIZE_BYTES + ))); + } + Ok(buf) +} + +/// Maximum size of a single `.prom` textfile that will be read +const MAX_TEXTFILE_SIZE_BYTES: u64 = 10 * 1024 * 1024; + +/// Build the `node_textfile_scrape_error` metric for a given file +fn textfile_scrape_error(filename: String, value: f64) -> CollectedMetric { + CollectedMetric { + name: "node_textfile_scrape_error".to_string(), + value, + labels: vec![("filename".to_string(), filename)], + metric_type: MetricType::Gauge, + unit: None, + description: Some("1 if there was an error scraping the textfile, 0 otherwise".to_string()), + } +} + +/// Strip a `# ` comment prefix, returning the remainder. +/// +/// The exposition format allows any run of whitespace after `#` and after the keyword, +/// so `#HELP`, `# HELP` and `# HELP` are all accepted. Returns `None` when the line is +/// not this kind of comment, so that plain comments fall through untouched. +fn strip_comment_keyword<'a>(line: &'a str, keyword: &str) -> Option<&'a str> { + let rest = line.strip_prefix('#')?.trim_start(); + let rest = rest.strip_prefix(keyword)?; + // The keyword must be followed by whitespace, so `# HELPfoo` is not a HELP line + if rest.starts_with(char::is_whitespace) { + Some(rest) + } else { + None + } +} + +/// Decode the escape sequences the Prometheus exposition format allows in HELP text +fn unescape_exposition(s: &str) -> String { + if !s.contains('\\') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('n') => out.push('\n'), + Some('\\') => out.push('\\'), + // Any other escape is passed through verbatim, as the format leaves it undefined + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + } + } + out +} + +/// Parse a single Prometheus exposition format line +/// Format: metric_name[{label="value",...}] value [timestamp] +fn parse_prometheus_line( + line: &str, + metric_types: &HashMap, +) -> Option { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + + // Find the metric name and optional labels + let (name, labels_str, value_str) = if let Some(brace_start) = line.find('{') { + // Has labels — find closing '}' while respecting quoted strings + let brace_end = find_closing_brace(&line[brace_start + 1..])?; + let brace_end = brace_start + 1 + brace_end; + // Whitespace between the name and `{` is tolerated by the exposition format + let name = line[..brace_start].trim_end(); + let labels_str = Some(&line[brace_start + 1..brace_end]); + let rest = line[brace_end + 1..].trim(); + (name, labels_str, rest) + } else { + // No labels - split on whitespace + let mut parts = line.splitn(2, char::is_whitespace); + let name = parts.next()?; + let rest = parts.next().unwrap_or("").trim(); + (name, None, rest) + }; + + // Reject names that are not valid metric identifiers — `.prom` content is + // user-supplied and flows straight into the OTLP metric name. + if !is_valid_metric_name(name) { + return None; + } + + // Parse the value, ignoring the optional trailing timestamp: as in Prometheus + // `node_exporter`, textfile samples are stamped at scrape time. + // `f64::from_str` already accepts the format's `+Inf`, `-Inf` and `NaN` spellings. + let value_str = value_str.split_whitespace().next()?; + let value: f64 = value_str.parse().ok()?; + + // Parse labels if present + let mut labels: Vec<(String, String)> = Vec::new(); + if let Some(labels_str) = labels_str { + for part in split_labels(labels_str) { + // Any malformed label fails the whole line rather than being dropped: losing a + // label changes the series identity, which would collapse distinct samples onto + // one attribute set with no diagnostic. The caller counts this as a parse + // failure and raises `node_textfile_scrape_error`. + let (key, val) = parse_label(part)?; + + // `.prom` content is user-supplied and flows straight into OTLP attribute keys, + // so apply the same grammar check as the metric name. + if !is_valid_label_name(&key) { + return None; + } + // Keep the first occurrence of a repeated key — two attributes sharing a key + // would make the data point invalid, and first-wins preserves identity. + if labels.iter().any(|(k, _)| *k == key) { + continue; + } + labels.push((key, val)); + } + } + + // Determine metric type (default to gauge) + let metric_type = metric_types.get(name).copied().unwrap_or(MetricType::Gauge); + + Some(CollectedMetric { + name: name.to_string(), + value, + labels, + metric_type, + unit: None, + description: None, + }) +} + +/// Check that a name matches the Prometheus label name grammar `[a-zA-Z_][a-zA-Z0-9_]*` +fn is_valid_label_name(name: &str) -> bool { + let mut chars = name.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' => {} + _ => return false, + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +/// Check that a name matches the Prometheus metric name grammar `[a-zA-Z_:][a-zA-Z0-9_:]*` +fn is_valid_metric_name(name: &str) -> bool { + let mut chars = name.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' || c == ':' => {} + _ => return false, + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == ':') +} + +/// Find the closing `}` in a label block, skipping over quoted strings and escapes. +/// Input starts right after the opening `{`. Returns the byte offset of `}` within `s`. +fn find_closing_brace(s: &str) -> Option { + let mut in_quotes = false; + let mut prev_backslash = false; + for (i, c) in s.char_indices() { + match c { + '\\' if in_quotes => { + prev_backslash = !prev_backslash; + continue; + } + '"' if !prev_backslash => in_quotes = !in_quotes, + '}' if !in_quotes => return Some(i), + _ => {} + } + prev_backslash = false; + } + None +} + +/// Split label string by commas, respecting quoted values and escape sequences +fn split_labels(labels_str: &str) -> Vec<&str> { + let mut parts = Vec::new(); + let mut start = 0; + let mut in_quotes = false; + let mut prev_backslash = false; + + for (i, c) in labels_str.char_indices() { + match c { + '\\' if in_quotes => { + prev_backslash = !prev_backslash; + continue; + } + '"' if !prev_backslash => in_quotes = !in_quotes, + ',' if !in_quotes => { + parts.push(&labels_str[start..i]); + start = i + 1; + } + _ => {} + } + prev_backslash = false; + } + + // Add the last part + if start < labels_str.len() { + parts.push(&labels_str[start..]); + } + + parts +} + +/// Parse a single label: key="value" +fn parse_label(label: &str) -> Option<(String, String)> { + let label = label.trim(); + let eq_pos = label.find('=')?; + let key = label[..eq_pos].trim(); + let val = label[eq_pos + 1..].trim(); + + // An empty key would become an empty OTLP attribute key + if key.is_empty() { + return None; + } + + // Remove exactly one surrounding quote from each end + let val = if val.starts_with('"') && val.ends_with('"') && val.len() >= 2 { + &val[1..val.len() - 1] + } else { + val + }; + + // Handle escape sequences in a single pass (avoids NUL-byte placeholder issues) + let mut result = String::with_capacity(val.len()); + let mut chars = val.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + match chars.next() { + Some('\\') => result.push('\\'), + Some('"') => result.push('"'), + Some('n') => result.push('\n'), + Some(other) => { + result.push('\\'); + result.push(other); + } + None => result.push('\\'), + } + } else { + result.push(c); + } + } + + Some((key.to_string(), result)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::receivers::node_metrics::collector::test_support::*; + use crate::receivers::node_metrics::config::Collector; + use std::fs::File; + use std::os::unix::fs::symlink; + + // --------------------------------------------------------------------- + // Textfile collector + // --------------------------------------------------------------------- + + #[test] + fn test_collect_textfile_directory_contents() { + let tmp = tempfile::tempdir().unwrap(); + // Created in the exact reverse of sorted filename order, so a collector that + // dropped the sort could only pass by accident: the OTLP conversion takes a + // metric's description and unit from the first sample it sees, and reorders + // data points between scrapes if traversal order decides which file wins. + std::fs::write(tmp.path().join("z.prom"), "shared_metric{job=\"z\"} 40\n").unwrap(); + std::fs::write( + tmp.path().join("m.prom"), + "\ +# HELP shared_metric Documented in m.prom +shared_metric{job=\"m\"} 30 +", + ) + .unwrap(); + std::fs::write( + tmp.path().join("b.prom"), + "\ +# HELP other_metric Documented in b.prom +shared_metric{job=\"b\"} 20 +other_metric 3.5 +", + ) + .unwrap(); + std::fs::write( + tmp.path().join("a.prom"), + "\ +# HELP shared_metric Documented in a.prom +# TYPE shared_metric counter +shared_metric{job=\"a\"} 10 +this line is not valid exposition format +", + ) + .unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let config = NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(tmp.path().to_str().unwrap().to_string()), + ..Default::default() + }; + + let metrics = collector.collect_textfile(&config); + + // Values, labels, types and HELP text all come from the file. + let from_a = find_one(&metrics, "shared_metric", &[("job", "a")]); + assert_eq!(from_a.value, 10.0); + assert_eq!(from_a.metric_type, MetricType::Counter); + assert_eq!(from_a.description.as_deref(), Some("Documented in a.prom")); + + // TYPE and HELP are scoped to the file that declares them. + let from_b = find_one(&metrics, "shared_metric", &[("job", "b")]); + assert_eq!(from_b.value, 20.0); + assert_eq!(from_b.metric_type, MetricType::Gauge); + assert_eq!(from_b.description, None); + + let other = find_one(&metrics, "other_metric", &[]); + assert_eq!(other.value, 3.5); + assert_eq!(other.description.as_deref(), Some("Documented in b.prom")); + + let from_m = find_one(&metrics, "shared_metric", &[("job", "m")]); + assert_eq!(from_m.value, 30.0); + assert_eq!(from_m.description.as_deref(), Some("Documented in m.prom")); + + let from_z = find_one(&metrics, "shared_metric", &[("job", "z")]); + assert_eq!(from_z.value, 40.0); + assert_eq!(from_z.description, None); + + // The malformed line marks only its own file as failed. + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "a.prom")] + ) + .value, + 1.0 + ); + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "b.prom")] + ) + .value, + 0.0 + ); + + // Modification times are exported so a stalled writer is detectable, and + // the files are visited in sorted filename order, not creation order. + let mtimes: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "node_textfile_mtime_seconds") + .map(|m| label(m, "filename")) + .collect(); + assert_eq!(mtimes, ["a.prom", "b.prom", "m.prom", "z.prom"]); + + // The samples themselves are emitted in that same order, so the description + // and unit the OTLP conversion keeps for `shared_metric` are always a.prom's. + let shared_order: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "shared_metric") + .map(|m| label(m, "job")) + .collect(); + assert_eq!(shared_order, ["a", "b", "m", "z"]); + + assert!( + find_one( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "a.prom")] + ) + .value + > 0.0 + ); + } + + #[test] + fn test_textfile_help_descriptions() { + let tmp = tempfile::tempdir().unwrap(); + let prom = tmp.path().join("test.prom"); + std::fs::write( + &prom, + "\ +# HELP my_gauge A helpful description for the gauge +# TYPE my_gauge gauge +my_gauge 42 +# HELP my_counter Total number of things +# TYPE my_counter counter +my_counter 100 +no_help_metric 7 +", + ) + .unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let config = NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(prom.to_str().unwrap().to_string()), + ..Default::default() + }; + + let metrics = collector.collect_textfile(&config); + + let find = |name: &str| metrics.iter().find(|m| m.name == name); + + // Metrics with HELP comments should have descriptions + let gauge = find("my_gauge").expect("my_gauge not found"); + assert_eq!( + gauge.description.as_deref(), + Some("A helpful description for the gauge") + ); + assert_eq!(gauge.metric_type, MetricType::Gauge); + + let counter = find("my_counter").expect("my_counter not found"); + assert_eq!( + counter.description.as_deref(), + Some("Total number of things") + ); + assert_eq!(counter.metric_type, MetricType::Counter); + + // Metric without HELP should have no description + let no_help = find("no_help_metric").expect("no_help_metric not found"); + assert_eq!(no_help.description, None); + } + + #[test] + fn test_textfile_single_file_path() { + let tmp = tempfile::tempdir().unwrap(); + let prom = tmp.path().join("single.prom"); + std::fs::write(&prom, "single_metric 99\n").unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let config = NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(prom.to_str().unwrap().to_string()), + ..Default::default() + }; + + let metrics = collector.collect_textfile(&config); + let found = metrics.iter().find(|m| m.name == "single_metric"); + assert!( + found.is_some(), + "single_metric not found in {:?}", + metrics.iter().map(|m| &m.name).collect::>() + ); + assert_eq!(found.unwrap().value, 99.0); + } + + #[test] + fn test_textfile_directory_path() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("a.prom"), "metric_a 1\n").unwrap(); + std::fs::write(tmp.path().join("b.prom"), "metric_b 2\n").unwrap(); + std::fs::write(tmp.path().join("ignored.txt"), "not_a_metric 3\n").unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let config = NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(tmp.path().to_str().unwrap().to_string()), + ..Default::default() + }; + + let metrics = collector.collect_textfile(&config); + let names: Vec<&str> = metrics.iter().map(|m| m.name.as_str()).collect(); + + assert!( + names.contains(&"metric_a"), + "metric_a not found in {:?}", + names + ); + assert!( + names.contains(&"metric_b"), + "metric_b not found in {:?}", + names + ); + assert!( + !names.contains(&"not_a_metric"), + ".txt file should be ignored" + ); + } + + /// The textfile config for a `.prom` file or a directory of them. + fn textfile_config(path: &str) -> NodeMetricsConfig { + NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(path.to_string()), + ..Default::default() + } + } + + /// A `.prom` that cannot be read — a dangling symlink is the realistic case — must be + /// reported, not silently skipped, or a deployment whose symlink target was garbage + /// collected loses metrics with no signal at all. + #[test] + fn test_textfile_dangling_symlink_is_reported_as_an_error() { + let tmp = tempfile::tempdir().unwrap(); + std::os::unix::fs::symlink( + tmp.path().join("no-such-target.prom"), + tmp.path().join("dangling.prom"), + ) + .unwrap(); + write(tmp.path(), "good.prom", "good_metric 1\n"); + + let collector = collector_with_empty_roots(); + let metrics = collector + .1 + .collect_textfile(&textfile_config(tmp.path().to_str().unwrap())); + + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "dangling.prom")] + ) + .value, + 1.0 + ); + // A dangling link has no mtime to report, and the healthy file is unaffected. + assert!( + matching( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "dangling.prom")] + ) + .is_empty() + ); + assert_eq!(find_one(&metrics, "good_metric", &[]).value, 1.0); + } + + /// A FIFO named `*.prom` must never be opened: `open(2)` on one with no writer blocks + /// forever, which would wedge the scrape thread and stop all node metrics. + #[test] + fn test_textfile_fifo_is_skipped_without_reading_it() { + let tmp = tempfile::tempdir().unwrap(); + let fifo = tmp.path().join("pipe.prom"); + let cpath = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap(); + // SAFETY: mkfifo on a path inside our own temp directory + let rc = unsafe { libc::mkfifo(cpath.as_ptr(), 0o600) }; + assert_eq!(rc, 0, "could not create fifo for the test"); + write(tmp.path(), "plain.prom", "plain_metric 7\n"); + + let collector = collector_with_empty_roots(); + // If the FIFO were opened this call would never return. + let metrics = collector + .1 + .collect_textfile(&textfile_config(tmp.path().to_str().unwrap())); + + assert_eq!(find_one(&metrics, "plain_metric", &[]).value, 7.0); + // Skipped silently: it is not a textfile at all, so it is not an error either. + assert!( + matching( + &metrics, + "node_textfile_scrape_error", + &[("filename", "pipe.prom")] + ) + .is_empty() + ); + } + + /// The exposition format allows any whitespace run after `#`, but the keyword itself + /// must be followed by whitespace — `# HELPfoo` is an ordinary comment. + #[test] + fn test_comment_keyword_whitespace_handling() { + let mut types = HashMap::new(); + types.insert("m".to_string(), MetricType::Counter); + + assert_eq!( + strip_comment_keyword("#HELP m desc", "HELP"), + Some(" m desc") + ); + assert_eq!( + strip_comment_keyword("# TYPE m counter", "TYPE"), + Some(" m counter") + ); + assert_eq!(strip_comment_keyword("# HELPfoo m desc", "HELP"), None); + assert_eq!(strip_comment_keyword("# TYPEfoo m counter", "TYPE"), None); + assert_eq!(strip_comment_keyword("# just a comment", "HELP"), None); + assert_eq!(strip_comment_keyword("not a comment", "HELP"), None); + assert_eq!(strip_comment_keyword("#", "HELP"), None); + } + + /// `.prom` content is user-supplied, so label keys get the same grammar check as metric + /// names. An invalid key fails the line — dropping just the label would change the + /// series identity and silently collapse distinct samples onto one attribute set. + #[test] + fn test_invalid_label_key_rejects_the_line_and_duplicates_are_deduped() { + let types = HashMap::new(); + + assert!(parse_prometheus_line(r#"m{bad-key="1"} 5"#, &types).is_none()); + assert!(parse_prometheus_line(r#"m{="1"} 5"#, &types).is_none()); + assert!(parse_prometheus_line(r#"m{1abc="1"} 5"#, &types).is_none()); + + // A colon is legal in a metric name but not in a label name + assert!(parse_prometheus_line(r#"m{a:b="1"} 5"#, &types).is_none()); + + // First occurrence of a repeated key wins, and the line still parses + let metric = parse_prometheus_line(r#"m{ok="first",ok="second"} 5"#, &types).unwrap(); + assert_eq!(metric.labels, vec![("ok".to_string(), "first".to_string())]); + } + + #[test] + fn test_textfile_oversized_file_is_skipped_and_flagged() { + let tmp = tempfile::tempdir().unwrap(); + + // A runaway writer script: reading this on every scrape interval would pull + // the whole file into memory. Created sparsely — only the first line is + // written, then the file is extended one byte past the cap. + let big = tmp.path().join("big.prom"); + std::fs::write(&big, "big_metric 1\n").unwrap(); + File::options() + .write(true) + .open(&big) + .unwrap() + .set_len(MAX_TEXTFILE_SIZE_BYTES + 1) + .unwrap(); + std::fs::write(tmp.path().join("small.prom"), "small_metric 2\n").unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let metrics = collector.collect_textfile(&textfile_config(tmp.path().to_str().unwrap())); + + // The oversized file is not parsed at all, and says so. + assert!(matching(&metrics, "big_metric", &[]).is_empty()); + let error = find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "big.prom")], + ); + assert_eq!(error.value, 1.0); + assert_eq!(error.metric_type, MetricType::Gauge); + assert_eq!(error.unit, None); + assert_eq!(error.labels.len(), 1); + + // Its modification time is still reported, so a writer stuck emitting an + // oversized file stays observable. + find_one( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "big.prom")], + ); + + // Skipping one file does not abandon the rest of the directory. + assert_eq!(find_one(&metrics, "small_metric", &[]).value, 2.0); + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "small.prom")] + ) + .value, + 0.0 + ); + } + + #[test] + fn test_textfile_missing_directory_reports_error_against_configured_path() { + let tmp = tempfile::tempdir().unwrap(); + let missing = tmp.path().join("no-such-directory"); + let configured = missing.to_str().unwrap().to_string(); + + let (_roots, collector) = collector_with_empty_roots(); + let metrics = collector.collect_textfile(&textfile_config(&configured)); + + // A mistyped directory must not fail silently, or the receiver just emits + // nothing forever. + assert_eq!(metrics.len(), 1); + let error = &metrics[0]; + assert_eq!(error.name, "node_textfile_scrape_error"); + assert_eq!(error.value, 1.0); + assert_eq!(error.metric_type, MetricType::Gauge); + assert_eq!(error.unit, None); + + // Deliberately asymmetric with the per-file errors, which carry a bare + // filename: there is no file here, so the configured path is the only thing + // that identifies what went wrong. + assert_eq!(error.labels, vec![("filename".to_string(), configured)]); + } + + #[test] + fn test_textfile_unsupported_type_falls_back_to_gauge() { + let tmp = tempfile::tempdir().unwrap(); + let prom = tmp.path().join("types.prom"); + std::fs::write( + &prom, + "\ +# TYPE my_histogram histogram +my_histogram 5 +# TYPE my_summary summary +my_summary 6 +# TYPE my_untyped untyped +my_untyped 7 +# TYPE my_counter counter +my_counter 8 +", + ) + .unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let metrics = collector.collect_textfile(&textfile_config(prom.to_str().unwrap())); + + // histogram, summary and untyped are not modelled; their series are exported + // as gauges rather than being dropped or promoted to counters. + for (name, value) in [ + ("my_histogram", 5.0), + ("my_summary", 6.0), + ("my_untyped", 7.0), + ] { + let metric = find_one(&metrics, name, &[]); + assert_eq!(metric.metric_type, MetricType::Gauge, "{}", name); + assert_eq!(metric.value, value, "{}", name); + } + + // A recognised type in the same file is still honoured. + let counter = find_one(&metrics, "my_counter", &[]); + assert_eq!(counter.metric_type, MetricType::Counter); + assert_eq!(counter.value, 8.0); + + // An unsupported TYPE is not a parse failure. + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "types.prom")] + ) + .value, + 0.0 + ); + } + + #[test] + fn test_textfile_symlinked_prom_file_is_collected() { + // The real file lives outside the scanned directory, as it does when a + // deployment symlinks generated metrics into place. + let elsewhere = tempfile::tempdir().unwrap(); + let real = elsewhere.path().join("generated.prom"); + std::fs::write(&real, "# TYPE linked_metric counter\nlinked_metric 5\n").unwrap(); + + let tmp = tempfile::tempdir().unwrap(); + symlink(&real, tmp.path().join("linked.prom")).unwrap(); + std::fs::write(tmp.path().join("plain.prom"), "plain_metric 1\n").unwrap(); + // A *directory* named like a textfile is still not a textfile. + std::fs::create_dir(tmp.path().join("adirectory.prom")).unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let metrics = collector.collect_textfile(&textfile_config(tmp.path().to_str().unwrap())); + + // The scan resolves symlinks, so the link is read like any other file. + let linked = find_one(&metrics, "linked_metric", &[]); + assert_eq!(linked.value, 5.0); + assert_eq!(linked.metric_type, MetricType::Counter); + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "linked.prom")] + ) + .value, + 0.0 + ); + // The link's own name is what identifies it, not the target's. + find_one( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "linked.prom")], + ); + assert!( + matching( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "generated.prom")] + ) + .is_empty() + ); + + assert_eq!(find_one(&metrics, "plain_metric", &[]).value, 1.0); + + // Exactly two files were scanned: the directory named `adirectory.prom` is + // not reported as an unreadable textfile. + let mut scanned: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "node_textfile_scrape_error") + .map(|m| label(m, "filename")) + .collect(); + scanned.sort_unstable(); + assert_eq!(scanned, ["linked.prom", "plain.prom"]); + } + + // --------------------------------------------------------------------- + // Prometheus exposition-format parsing + // --------------------------------------------------------------------- + + #[test] + fn test_parse_prometheus_line_simple() { + let types = HashMap::new(); + let metric = parse_prometheus_line("my_metric 42.5", &types).unwrap(); + assert_eq!(metric.name, "my_metric"); + assert_eq!(metric.value, 42.5); + assert!(metric.labels.is_empty()); + assert_eq!(metric.metric_type, MetricType::Gauge); + } + + #[test] + fn test_parse_prometheus_line_with_labels() { + let types = HashMap::new(); + let metric = parse_prometheus_line( + "http_requests_total{method=\"GET\",path=\"/api\"} 1234", + &types, + ) + .unwrap(); + assert_eq!(metric.name, "http_requests_total"); + assert_eq!(metric.value, 1234.0); + let find_label = |key: &str| { + metric + .labels + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.clone()) + }; + assert_eq!(find_label("method"), Some("GET".to_string())); + assert_eq!(find_label("path"), Some("/api".to_string())); + } + + #[test] + fn test_parse_prometheus_line_with_type() { + let mut types = HashMap::new(); + types.insert("my_counter".to_string(), MetricType::Counter); + + let metric = parse_prometheus_line("my_counter 100", &types).unwrap(); + assert_eq!(metric.metric_type, MetricType::Counter); + } + + #[test] + fn test_parse_prometheus_line_special_values() { + let types = HashMap::new(); + + let inf = parse_prometheus_line("metric_inf +Inf", &types).unwrap(); + assert!(inf.value.is_infinite() && inf.value.is_sign_positive()); + + let neg_inf = parse_prometheus_line("metric_neginf -Inf", &types).unwrap(); + assert!(neg_inf.value.is_infinite() && neg_inf.value.is_sign_negative()); + + let nan = parse_prometheus_line("metric_nan NaN", &types).unwrap(); + assert!(nan.value.is_nan()); + } + + #[test] + fn test_parse_prometheus_line_with_timestamp() { + let types = HashMap::new(); + // Timestamp should be ignored + let metric = parse_prometheus_line("my_metric 42.5 1395066363000", &types).unwrap(); + assert_eq!(metric.name, "my_metric"); + assert_eq!(metric.value, 42.5); + } + + #[test] + fn test_parse_prometheus_line_escaped_label() { + let types = HashMap::new(); + let metric = + parse_prometheus_line("metric{label=\"value with \\\"quotes\\\"\"} 1", &types).unwrap(); + let label_val = metric + .labels + .iter() + .find(|(k, _)| k == "label") + .map(|(_, v)| v.clone()); + assert_eq!(label_val, Some("value with \"quotes\"".to_string())); + } + + #[test] + fn test_parse_prometheus_line_brace_in_label_value() { + let types = HashMap::new(); + let metric = parse_prometheus_line(r#"metric{label="a}b"} 42"#, &types).unwrap(); + assert_eq!(metric.name, "metric"); + assert_eq!(metric.value, 42.0); + let label_val = metric + .labels + .iter() + .find(|(k, _)| k == "label") + .map(|(_, v)| v.clone()); + assert_eq!(label_val, Some("a}b".to_string())); + } + + #[test] + fn test_parse_prometheus_line_empty() { + let types = HashMap::new(); + assert!(parse_prometheus_line("", &types).is_none()); + assert!(parse_prometheus_line("# comment", &types).is_none()); + } + + #[test] + fn test_split_labels() { + let labels = split_labels("method=\"GET\",path=\"/api\""); + assert_eq!(labels.len(), 2); + assert_eq!(labels[0], "method=\"GET\""); + assert_eq!(labels[1], "path=\"/api\""); + } + + #[test] + fn test_split_labels_with_comma_in_value() { + let labels = split_labels("label=\"value,with,commas\",other=\"ok\""); + assert_eq!(labels.len(), 2); + assert_eq!(labels[0], "label=\"value,with,commas\""); + assert_eq!(labels[1], "other=\"ok\""); + } + + #[test] + fn test_split_labels_with_escaped_quotes() { + let labels = split_labels(r#"method="GET",path="/foo\"bar""#); + assert_eq!(labels.len(), 2); + assert_eq!(labels[0], r#"method="GET""#); + assert_eq!(labels[1], r#"path="/foo\"bar""#); + } + + #[test] + fn test_split_labels_with_escaped_backslash_before_quote() { + // \\\" in the source: escaped backslash followed by escaped quote + let labels = split_labels(r#"a="x\\\"y",b="ok""#); + assert_eq!(labels.len(), 2); + assert_eq!(labels[0], r#"a="x\\\"y""#); + assert_eq!(labels[1], r#"b="ok""#); + } + + #[test] + fn test_parse_label_escaped_backslash_before_n() { + // \\n in exposition format = literal backslash followed by 'n' + let result = parse_label(r#"key="a\\nb""#).unwrap(); + assert_eq!(result.0, "key"); + assert_eq!(result.1, "a\\nb"); + } + + #[test] + fn test_parse_label_unknown_escape() { + // Unknown escape sequences should preserve the backslash + let result = parse_label(r#"key="a\tb""#).unwrap(); + assert_eq!(result.1, "a\\tb"); + } +} diff --git a/src/receivers/node_metrics/collector/util.rs b/src/receivers/node_metrics/collector/util.rs new file mode 100644 index 00000000..2260063c --- /dev/null +++ b/src/receivers/node_metrics/collector/util.rs @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Path, device-name and `utsname` field helpers shared by the collectors. +//! +//! The virtual-device and partition exclusion heuristics are derived from Prometheus +//! node_exporter — see the notice in this module's `mod.rs`. + +/// Check if `path` matches a prefix with path-boundary awareness. +/// +/// Returns `true` when `path` equals `prefix` exactly or starts with `prefix` +/// followed by `/`. This prevents `/dev` from matching `/developer`. +/// +/// A trailing `/` on the prefix is ignored, so `/dev` and `/dev/` behave alike. +pub(crate) fn matches_path_prefix(path: &str, prefix: &str) -> bool { + // "/" trims to "", and the root prefix matches every absolute path + let trimmed = prefix.trim_end_matches('/'); + if trimmed.is_empty() { + return prefix.starts_with('/') && path.starts_with('/'); + } + path == trimmed + || path.starts_with(trimmed) && path.as_bytes().get(trimmed.len()) == Some(&b'/') +} + +/// Check if a device name is a virtual block device that should not be reported +/// +/// Covers the `z?ram`, `loop` and `fd` families of node_exporter's default device +/// exclusion (its full default also excludes partitions, which `is_partition` handles). +pub(super) fn is_virtual_disk(device: &str) -> bool { + for prefix in &["zram", "ram", "loop", "fd"] { + if let Some(rest) = device.strip_prefix(prefix) + && !rest.is_empty() + && rest.chars().all(|c| c.is_ascii_digit()) + { + return true; + } + } + false +} + +/// Check if a device name looks like a partition +/// +/// This is a name heuristic. Partitions are excluded from disk I/O metrics so their +/// counters are not double-counted alongside the whole disk that contains them. +/// +/// Deliberately stricter than node_exporter, whose default `p` rule applies only +/// to NVMe: this also treats `mmcblk0p1`, `md127p1`, `nbd0p1`, `zd16p1`, `pmem0p1` and +/// `dasda1` as partitions, so those per-partition series are not reported. +pub(super) fn is_partition(device: &str) -> bool { + // Devices whose whole-disk name ends in a digit delimit the partition with `p`: + // nvme0n1p1, mmcblk0p1, md127p1, nbd0p1, zd16p1, pmem0p1, loop0p1. + if let Some((disk, part)) = device.rsplit_once('p') + && !part.is_empty() + && part.chars().all(|c| c.is_ascii_digit()) + && disk.ends_with(|c: char| c.is_ascii_digit()) + { + return true; + } + + // SCSI/SATA/virtio/s390 partitions: sda1, vda1, xvda1, hda1, sdaa1, dasda1. + // After the prefix, one or more lowercase letters identify the disk, + // and any trailing digits indicate a partition number. + for prefix in &["sd", "vd", "xvd", "hd", "dasd"] { + if let Some(rest) = device.strip_prefix(prefix) { + let letter_count = rest.chars().take_while(|c| c.is_ascii_lowercase()).count(); + if letter_count == 0 { + continue; + } + let after_letters = &rest[letter_count..]; + if !after_letters.is_empty() && after_letters.chars().all(|c| c.is_ascii_digit()) { + return true; + } + } + } + + false +} + +/// Decode octal escape sequences from /proc/mounts paths. +/// +/// The kernel encodes space as `\040`, tab as `\011`, newline as `\012`, +/// and backslash as `\134` in device and mount-point fields. +pub(super) fn decode_mount_path(s: &str) -> String { + // Fast path: no escapes present (common case) + if !s.contains('\\') { + return s.to_string(); + } + let bytes = s.as_bytes(); + let mut result = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\\' + && i + 3 < bytes.len() + && (b'0'..=b'7').contains(&bytes[i + 1]) + && (b'0'..=b'7').contains(&bytes[i + 2]) + && (b'0'..=b'7').contains(&bytes[i + 3]) + { + let val = (bytes[i + 1] - b'0') as u16 * 64 + + (bytes[i + 2] - b'0') as u16 * 8 + + (bytes[i + 3] - b'0') as u16; + // Three octal digits can represent up to \777 = 511, so + // guard against values that don't fit in a single byte. + // The kernel only emits valid byte values, but be safe. + if val <= 255 { + result.push(val as u8); + i += 4; + } else { + result.push(bytes[i]); + i += 1; + } + } else { + result.push(bytes[i]); + i += 1; + } + } + String::from_utf8(result).unwrap_or_else(|_| s.to_string()) +} + +/// Convert a fixed-size C character array from `utsname` into a Rust string +/// +/// `uname(2)` NUL-terminates each field, but the scan is bounded by the array length so +/// that a field which is not terminated cannot be read out of bounds. +pub(super) fn utsname_field(field: &[libc::c_char]) -> String { + let bytes: Vec = field + .iter() + .take_while(|&&c| c != 0) + .map(|&c| c as u8) + .collect(); + String::from_utf8_lossy(&bytes).to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + // --------------------------------------------------------------------- + // Device-name classification + // --------------------------------------------------------------------- + + #[test] + fn test_is_virtual_disk() { + // Virtual devices: an accepted prefix followed by nothing but digits, + // equivalent to node_exporter's `^(z?ram|loop|fd)\d+$`. + assert!(is_virtual_disk("ram0")); + assert!(is_virtual_disk("ram15")); + assert!(is_virtual_disk("zram0")); + assert!(is_virtual_disk("loop0")); + assert!(is_virtual_disk("loop10")); + assert!(is_virtual_disk("fd0")); + + // Plain prefix matching would wrongly exclude these: the suffix must be + // digits only, and a bare prefix with no index is not a device instance. + assert!(!is_virtual_disk("ramdisk0")); + assert!(!is_virtual_disk("loopback")); + assert!(!is_virtual_disk("ram")); + assert!(!is_virtual_disk("zram")); + assert!(!is_virtual_disk("loop")); + assert!(!is_virtual_disk("fd")); + assert!(!is_virtual_disk("loop0p1")); + + // Real disks are never excluded. + assert!(!is_virtual_disk("sda")); + assert!(!is_virtual_disk("nvme0n1")); + assert!(!is_virtual_disk("mmcblk0")); + } + + #[test] + fn test_is_partition() { + // Whole disks + assert!(!is_partition("sda")); + assert!(!is_partition("sdb")); + assert!(!is_partition("vda")); + assert!(!is_partition("xvda")); + assert!(!is_partition("nvme0n1")); + assert!(!is_partition("hda")); + + assert!(!is_partition("mmcblk0")); + assert!(!is_partition("mmcblk1")); + + // Multi-letter disk identifiers (e.g. 27th+ disk) + assert!(!is_partition("sdaa")); + assert!(!is_partition("sdab")); + + // Whole disks whose names end in a digit but delimit no partition + assert!(!is_partition("dm-0")); + assert!(!is_partition("md0")); + assert!(!is_partition("md127")); + assert!(!is_partition("sr0")); + assert!(!is_partition("loop0")); + + // Partitions + assert!(is_partition("sda1")); + assert!(is_partition("sda12")); + assert!(is_partition("sdaa1")); + assert!(is_partition("sdaa12")); + assert!(is_partition("vda1")); + assert!(is_partition("nvme0n1p1")); + assert!(is_partition("nvme0n1p12")); + assert!(is_partition("xvda1")); + assert!(is_partition("mmcblk0p1")); + assert!(is_partition("mmcblk0p12")); + assert!(is_partition("md127p1")); + assert!(is_partition("dasda1")); + } + + // --------------------------------------------------------------------- + // Path decoding and prefix matching + // --------------------------------------------------------------------- + + #[test] + fn test_decode_mount_path_no_escapes() { + assert_eq!(decode_mount_path("/mnt/data"), "/mnt/data"); + } + + #[test] + fn test_decode_mount_path_space() { + // \040 is space + assert_eq!(decode_mount_path("/mnt/my\\040drive"), "/mnt/my drive"); + } + + #[test] + fn test_decode_mount_path_multiple_escapes() { + // \040 space, \011 tab, \012 newline, \134 backslash + assert_eq!(decode_mount_path("/mnt/a\\040b\\134c"), "/mnt/a b\\c"); + } + + #[test] + fn test_decode_mount_path_tab_and_newline() { + assert_eq!(decode_mount_path("a\\011b\\012c"), "a\tb\nc"); + } + + #[test] + fn test_decode_mount_path_trailing_backslash() { + // Incomplete escape at end should be preserved as-is + assert_eq!(decode_mount_path("/mnt/foo\\04"), "/mnt/foo\\04"); + } + + #[test] + fn test_decode_mount_path_non_octal_after_backslash() { + // \9 is not a valid octal digit, should be preserved + assert_eq!(decode_mount_path("/mnt/foo\\9bar"), "/mnt/foo\\9bar"); + } + + #[test] + fn test_matches_path_prefix_exact() { + assert!(matches_path_prefix("/dev", "/dev")); + assert!(matches_path_prefix("/proc", "/proc")); + } + + #[test] + fn test_matches_path_prefix_subpath() { + assert!(matches_path_prefix("/dev/shm", "/dev")); + assert!(matches_path_prefix("/proc/sys/fs", "/proc")); + assert!(matches_path_prefix( + "/var/lib/docker/overlay2", + "/var/lib/docker" + )); + assert!(matches_path_prefix( + "/var/lib/containers/storage/overlay", + "/var/lib/containers" + )); + } + + #[test] + fn test_matches_path_prefix_no_false_positives() { + // Must not match when the prefix is only a substring, not a path boundary + assert!(!matches_path_prefix("/developer", "/dev")); + assert!(!matches_path_prefix("/devops", "/dev")); + assert!(!matches_path_prefix("/processing", "/proc")); + assert!(!matches_path_prefix( + "/var/lib/dockerfiles", + "/var/lib/docker" + )); + assert!(!matches_path_prefix( + "/var/lib/containers2", + "/var/lib/containers" + )); + } + + #[test] + fn test_matches_path_prefix_no_match() { + assert!(!matches_path_prefix("/mnt/data", "/dev")); + assert!(!matches_path_prefix("/home/user", "/proc")); + } + + #[test] + fn test_matches_path_prefix_root_and_empty_prefix() { + // The root prefix trims to the empty string, so it cannot be compared as a + // path component: it has to match every absolute path instead. + assert!(matches_path_prefix("/", "/")); + assert!(matches_path_prefix("/x", "/")); + assert!(matches_path_prefix("/var/lib/docker", "/")); + + // An empty prefix is not the root and must match nothing, otherwise a blank + // configured exclusion would silently drop every filesystem. + assert!(!matches_path_prefix("/x", "")); + assert!(!matches_path_prefix("", "")); + + // A relative path is not below the root prefix. + assert!(!matches_path_prefix("x", "/")); + } + + #[test] + fn test_matches_path_prefix_ignores_trailing_slash_on_prefix() { + // A trailing slash must behave exactly like no trailing slash, so a + // configured "/dev/" is not silently inert. + for (path, expected) in [ + ("/dev", true), + ("/dev/shm", true), + ("/developer", false), + ("/mnt/data", false), + ] { + assert_eq!( + matches_path_prefix(path, "/dev/"), + expected, + "prefix /dev/ against {}", + path + ); + assert_eq!( + matches_path_prefix(path, "/dev/"), + matches_path_prefix(path, "/dev"), + "trailing slash changed the result for {}", + path + ); + } + } + + // --------------------------------------------------------------------- + // utsname fields + // --------------------------------------------------------------------- + + /// Build a `utsname`-style field from raw bytes. + /// + /// `libc::c_char` is signed on most targets but unsigned on others (aarch64 Linux), + /// so the cast rather than a literal type keeps this portable. + fn utsname_bytes(bytes: &[u8]) -> Vec { + bytes.iter().map(|&b| b as libc::c_char).collect() + } + + #[test] + fn test_utsname_field_stops_at_nul() { + // uname(2) NUL-terminates each field and leaves the remainder of the array + // undefined; everything past the terminator must be ignored. + let field = utsname_bytes(b"Linux\0-leftover-garbage\0"); + assert_eq!(utsname_field(&field), "Linux"); + + // An empty field is an empty string, not a stray NUL character. + assert_eq!(utsname_field(&utsname_bytes(b"\0\0\0")), ""); + assert_eq!(utsname_field(&[]), ""); + } + + #[test] + fn test_utsname_field_without_terminator_truncates_at_array_length() { + // A field that fills the array with no room for the terminator must stop at + // the end of the slice rather than reading past it. + let field = utsname_bytes(b"6.1.0-no-nul-here"); + assert_eq!(utsname_field(&field), "6.1.0-no-nul-here"); + assert_eq!(utsname_field(&field).len(), field.len()); + } + + #[test] + fn test_utsname_field_converts_non_utf8_lossily() { + // The kernel does not promise UTF-8 (a hostname can hold arbitrary bytes), and + // a lossy conversion is preferable to dropping the whole label. + let field = utsname_bytes(&[b'a', 0xFF, b'b', 0]); + assert_eq!(utsname_field(&field), "a\u{FFFD}b"); + + // A truncated multi-byte sequence is replaced, not silently dropped. + assert_eq!(utsname_field(&utsname_bytes(&[0xC3, 0x28, 0])), "\u{FFFD}("); + } +} diff --git a/src/receivers/node_metrics/config.rs b/src/receivers/node_metrics/config.rs new file mode 100644 index 00000000..0abfd946 --- /dev/null +++ b/src/receivers/node_metrics/config.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Configuration for the node metrics receiver + +use std::time::Duration; + +/// Default procfs mount point +pub const DEFAULT_PROCFS_PATH: &str = "/proc"; +/// Default sysfs mount point +pub const DEFAULT_SYSFS_PATH: &str = "/sys"; + +/// Available metric collectors +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Collector { + /// CPU metrics (seconds per mode: user, system, idle, iowait, etc.) + Cpu, + /// Load average metrics (node_load1, node_load5, node_load15) + Loadavg, + /// Memory metrics (all fields from /proc/meminfo) + Memory, + /// Network interface metrics (bytes/packets rx/tx per interface) + Network, + /// Filesystem metrics (size/free/avail space per mount) + Filesystem, + /// System information (uname: kernel, hostname) + Uname, + /// Kernel stat counters (forks, context switches, interrupts, procs running/blocked) from /proc/stat + Stat, + /// Process metrics (kernel limits: threads-max, pid_max) from /proc/sys/kernel/ + Processes, + /// Disk I/O statistics from /proc/diskstats + Diskstats, + /// Virtual memory statistics from /proc/vmstat + Vmstat, + /// Network statistics from /proc/net/netstat and /proc/net/snmp + Netstat, + /// Socket statistics from /proc/net/sockstat and /proc/net/sockstat6 + Sockstat, + /// File descriptor statistics from /proc/sys/fs/file-nr + Filefd, + /// CPU frequency from /sys/devices/system/cpu/cpu*/cpufreq/ + Cpufreq, + /// Thermal zone temperatures from /sys/class/thermal/thermal_zone*/ + ThermalZone, + /// NVMe device info from /sys/class/nvme/ + Nvme, + /// Hardware monitoring sensors from /sys/class/hwmon/ + Hwmon, + /// Custom metrics from Prometheus-format text files + Textfile, + /// System time (node_time_seconds, node_boot_time_seconds) + Time, +} + +/// Configuration for the node metrics receiver +/// +/// This struct cannot derive `Deserialize` because `regex::Regex` does not implement it. +/// Supporting file-based configuration (YAML/TOML) would mean adding a raw config struct +/// with `String` filter fields that validates into this one. +#[derive(Clone, Debug)] +pub struct NodeMetricsConfig { + /// Interval between metric collections + pub scrape_interval: Duration, + /// Which collectors to enable + pub collectors: Vec, + /// Optional regex pattern to include only matching metric names + pub metric_include_filter: Option, + /// Optional regex pattern to exclude matching metric names + pub metric_exclude_filter: Option, + /// Optional regex pattern to exclude matching filesystem mount points, + /// in addition to the built-in virtual and network filesystem exclusions + pub filesystem_mount_exclude: Option, + /// Path to procfs mount (default: /proc) + pub procfs_path: String, + /// Path to sysfs mount (default: /sys) + pub sysfs_path: String, + /// Prefix under which the host's root filesystem is reachable, or "/" when the + /// collector runs directly on the host. + /// + /// Needed by the filesystem collector: the mount *table* can be read from a + /// bind-mounted host procfs, but `statfs` still has to be called on a path that + /// resolves in this process's own mount namespace. + pub rootfs_path: String, + /// Directory containing Prometheus-format textfiles (*.prom), or a single .prom file + pub textfile_directory: Option, + /// Service name used in the OTLP Resource (default: "node_metrics") + pub service_name: String, +} + +impl Default for NodeMetricsConfig { + fn default() -> Self { + Self { + scrape_interval: Duration::from_secs(60), + collectors: vec![ + Collector::Cpu, + Collector::Loadavg, + Collector::Memory, + Collector::Network, + Collector::Filesystem, + Collector::Uname, + Collector::Stat, + Collector::Processes, + Collector::Diskstats, + Collector::Vmstat, + Collector::Netstat, + Collector::Sockstat, + Collector::Filefd, + Collector::Cpufreq, + Collector::ThermalZone, + Collector::Nvme, + Collector::Hwmon, + Collector::Time, + // Textfile not included by default - requires a directory to be set + ], + metric_include_filter: None, + metric_exclude_filter: None, + filesystem_mount_exclude: None, + procfs_path: DEFAULT_PROCFS_PATH.to_string(), + sysfs_path: DEFAULT_SYSFS_PATH.to_string(), + rootfs_path: "/".to_string(), + textfile_directory: None, + service_name: "node_metrics".to_string(), + } + } +} + +impl NodeMetricsConfig { + /// Create a new config with the specified scrape interval in seconds + pub fn new(scrape_interval_secs: u64) -> Self { + Self { + scrape_interval: Duration::from_secs(scrape_interval_secs), + ..Default::default() + } + } + + /// Set which collectors to enable + pub fn with_collectors(mut self, collectors: Vec) -> Self { + self.collectors = collectors; + self + } + + /// Set the metric include filter regex pattern + pub fn with_include_filter(mut self, pattern: &str) -> Result { + self.metric_include_filter = Some(regex::Regex::new(pattern)?); + Ok(self) + } + + /// Set the metric exclude filter regex pattern + pub fn with_exclude_filter(mut self, pattern: &str) -> Result { + self.metric_exclude_filter = Some(regex::Regex::new(pattern)?); + Ok(self) + } + + /// Set the filesystem mount point exclude regex pattern + pub fn with_filesystem_mount_exclude(mut self, pattern: &str) -> Result { + self.filesystem_mount_exclude = Some(regex::Regex::new(pattern)?); + Ok(self) + } + + /// Check if a metric name passes the configured filters + pub fn should_include_metric(&self, name: &str) -> bool { + // If include filter is set, metric must match it + if let Some(ref include) = self.metric_include_filter + && !include.is_match(name) + { + return false; + } + + // If exclude filter is set, metric must not match it + if let Some(ref exclude) = self.metric_exclude_filter + && exclude.is_match(name) + { + return false; + } + + true + } + + /// Validate the configuration and normalize collector list (deduplicate) + pub fn normalize_and_validate(&mut self) -> Result<(), String> { + if self.scrape_interval < Duration::from_secs(1) { + return Err(format!( + "Scrape interval must be at least 1 second, got {:?}", + self.scrape_interval + )); + } + + if self.collectors.is_empty() { + return Err("At least one collector must be enabled".to_string()); + } + + // Deduplicate collectors while preserving order + let mut seen = std::collections::HashSet::new(); + self.collectors.retain(|c| seen.insert(c.clone())); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The collector set enabled by default (textfile requires an explicit directory) + const DEFAULT_COLLECTORS: [Collector; 18] = [ + Collector::Cpu, + Collector::Loadavg, + Collector::Memory, + Collector::Network, + Collector::Filesystem, + Collector::Uname, + Collector::Stat, + Collector::Processes, + Collector::Diskstats, + Collector::Vmstat, + Collector::Netstat, + Collector::Sockstat, + Collector::Filefd, + Collector::Cpufreq, + Collector::ThermalZone, + Collector::Nvme, + Collector::Hwmon, + Collector::Time, + ]; + + #[test] + fn test_default_config() { + let config = NodeMetricsConfig::default(); + assert_eq!(config.scrape_interval, Duration::from_secs(60)); + // Compare the actual set, not just its size: a count assertion would still pass + // if one collector were swapped for another. + assert_eq!(config.collectors, DEFAULT_COLLECTORS.to_vec()); + assert!(config.metric_include_filter.is_none()); + assert!(config.metric_exclude_filter.is_none()); + assert!(!config.collectors.contains(&Collector::Textfile)); + } + + #[test] + fn test_normalize_deduplicates_collectors() { + let mut config = NodeMetricsConfig::default().with_collectors(vec![ + Collector::Cpu, + Collector::Memory, + Collector::Cpu, + ]); + config.normalize_and_validate().unwrap(); + assert_eq!(config.collectors, vec![Collector::Cpu, Collector::Memory]); + } + + #[test] + fn test_validate_rejects_sub_second_interval_and_reports_it() { + let mut config = NodeMetricsConfig { + scrape_interval: Duration::from_millis(500), + ..Default::default() + }; + let err = config.normalize_and_validate().unwrap_err(); + // The rejected value must be reported accurately, not truncated to "0s" + assert!( + err.contains("500ms"), + "error should report the offending duration, got: {}", + err + ); + } + + #[test] + fn test_validate_rejects_empty_collector_list() { + let mut config = NodeMetricsConfig::default().with_collectors(vec![]); + assert!(config.normalize_and_validate().is_err()); + } + + #[test] + fn test_exclude_filter_drops_matching_metrics() { + let config = NodeMetricsConfig::default() + .with_exclude_filter("^node_vmstat_") + .unwrap(); + assert!(!config.should_include_metric("node_vmstat_pgfault")); + assert!(config.should_include_metric("node_load1")); + } + + #[test] + fn test_include_filter_takes_precedence_over_unmatched() { + let config = NodeMetricsConfig::default() + .with_include_filter("^node_load") + .unwrap(); + assert!(config.should_include_metric("node_load1")); + assert!(!config.should_include_metric("node_cpu_seconds_total")); + } + + #[test] + fn test_config_with_collectors() { + let config = + NodeMetricsConfig::new(30).with_collectors(vec![Collector::Loadavg, Collector::Memory]); + assert_eq!(config.scrape_interval, Duration::from_secs(30)); + assert_eq!( + config.collectors, + vec![Collector::Loadavg, Collector::Memory] + ); + } + + #[test] + fn test_combined_filters() { + let config = NodeMetricsConfig::new(60) + .with_include_filter("node_.*") + .unwrap() + .with_exclude_filter(".*packets.*") + .unwrap(); + + assert!(config.should_include_metric("node_load1")); + assert!(config.should_include_metric("node_network_receive_bytes_total")); + assert!(!config.should_include_metric("node_network_receive_packets_total")); + assert!(!config.should_include_metric("other_metric")); + } + + #[test] + fn test_filesystem_mount_exclude() { + let config = NodeMetricsConfig::new(60) + .with_filesystem_mount_exclude("^/nfs/") + .unwrap(); + + let re = config.filesystem_mount_exclude.as_ref().unwrap(); + assert!(re.is_match("/nfs/share")); + assert!(!re.is_match("/mnt/data")); + } +} diff --git a/src/receivers/node_metrics/convert.rs b/src/receivers/node_metrics/convert.rs new file mode 100644 index 00000000..97f36ea0 --- /dev/null +++ b/src/receivers/node_metrics/convert.rs @@ -0,0 +1,684 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Convert collected system metrics to OTLP ResourceMetrics format + +use crate::receivers::node_metrics::collector::{CollectedMetric, MetricType}; +use gethostname::gethostname; +use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value}; +use opentelemetry_proto::tonic::metrics::v1::{ + AggregationTemporality, Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, + metric, number_data_point, +}; +use opentelemetry_proto::tonic::resource::v1::Resource; +use std::collections::{BTreeMap, HashSet}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const SCOPE_NAME: &str = "node_metrics"; + +// Resource attributes +const SERVICE_NAME_KEY: &str = "service.name"; +const HOST_NAME_KEY: &str = "host.name"; +#[cfg(target_os = "linux")] +const OS_TYPE_KEY: &str = "os.type"; +#[cfg(target_os = "linux")] +const OS_TYPE_VALUE: &str = "linux"; + +/// Convert a batch of collected metrics to OTLP ResourceMetrics +/// +/// `boot_time_secs` is the system boot time in seconds since epoch, used as +/// the start time for cumulative counters. +/// +/// `service_name` is the value set on the `service.name` resource attribute. +pub fn convert_to_otlp_metrics( + metrics: Vec, + boot_time_secs: u64, + service_name: &str, +) -> ResourceMetrics { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let now_ns = now.as_secs() * 1_000_000_000 + now.subsec_nanos() as u64; + + let boot_time_ns = boot_time_secs.saturating_mul(1_000_000_000); + + // Group metrics by name *and* type so each OTLP Metric contains all its data points. + // Keying on the type as well keeps a gauge and a counter that happen to share a name + // (reachable via textfiles, whose `# TYPE` scope is per file) as two separate + // metrics, rather than silently stamping one group's points with the other's + // temporality and start time. + // BTreeMap gives deterministic (alphabetical) ordering across scrapes, + // which simplifies debugging and diffing. + let mut grouped: BTreeMap<(String, MetricType), Vec> = BTreeMap::new(); + for mut m in metrics { + // The name moves into the key; the grouped metrics keep an empty `name` field, + // which nothing below reads (only labels, value, unit and description are used). + let key = (std::mem::take(&mut m.name), m.metric_type); + grouped.entry(key).or_default().push(m); + } + + // A name that appears with two types stays two OTLP metrics, which a backend keying + // on resource+scope+name still treats as a conflict. Surface it rather than letting + // it be resolved arbitrarily downstream. + let mut previous: Option<&String> = None; + for (name, _) in grouped.keys() { + if previous == Some(name) { + // Debug, not warn: a conflicting `# TYPE` persists until the file is fixed, + // so warning would log this on every scrape forever. The same reasoning the + // textfile collector applies to its own repeating failures. + tracing::debug!( + "Metric {} is reported as both a gauge and a counter; check for a \ + conflicting `# TYPE` in the textfile directory", + name + ); + } + previous = Some(name); + } + + let otlp_metrics: Vec = grouped + .into_iter() + .map(|((name, metric_type), group)| { + convert_metric_group(name, metric_type, group, now_ns, boot_time_ns) + }) + .collect(); + + ResourceMetrics { + resource: Some(build_resource(service_name)), + scope_metrics: vec![ScopeMetrics { + scope: Some( + opentelemetry_proto::tonic::common::v1::InstrumentationScope { + name: SCOPE_NAME.to_string(), + version: String::new(), + attributes: vec![], + dropped_attributes_count: 0, + }, + ), + metrics: otlp_metrics, + schema_url: String::new(), + }], + schema_url: String::new(), + } +} + +/// Count the data points in a converted batch +/// +/// Every variant is matched explicitly: a silent `_ => 0` would under-count without any +/// compile error if a new metric shape were ever emitted here. +pub(crate) fn count_data_points(resource_metrics: &ResourceMetrics) -> usize { + resource_metrics + .scope_metrics + .iter() + .flat_map(|sm| sm.metrics.iter()) + .map(|m| match &m.data { + Some(metric::Data::Gauge(g)) => g.data_points.len(), + Some(metric::Data::Sum(s)) => s.data_points.len(), + Some(metric::Data::Histogram(h)) => h.data_points.len(), + Some(metric::Data::ExponentialHistogram(h)) => h.data_points.len(), + Some(metric::Data::Summary(s)) => s.data_points.len(), + None => 0, + }) + .sum() +} + +/// Build the OTLP Resource identifying the host these metrics describe +/// +/// `host.name` matters here: without it, the same series from different hosts share an +/// identical identity at the backend. Mirrors the kmsg receiver's resource attributes. +fn build_resource(service_name: &str) -> Resource { + let mut attributes = vec![string_attribute(SERVICE_NAME_KEY, service_name.to_string())]; + + if let Ok(hostname) = gethostname().into_string() { + attributes.push(string_attribute(HOST_NAME_KEY, hostname)); + } + + #[cfg(target_os = "linux")] + attributes.push(string_attribute(OS_TYPE_KEY, OS_TYPE_VALUE.to_string())); + + Resource { + attributes, + dropped_attributes_count: 0, + entity_refs: vec![], + } +} + +/// Build a string-valued OTLP attribute +fn string_attribute(key: &str, value: String) -> KeyValue { + KeyValue { + key: key.to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(value)), + }), + + key_strindex: 0, + } +} + +/// Convert a group of collected metrics (same name and type) to a single OTLP Metric +fn convert_metric_group( + name: String, + metric_type: MetricType, + group: Vec, + time_unix_nano: u64, + boot_time_nano: u64, +) -> Metric { + // Take the remaining metadata from the first metric in the group + let (description, unit) = match group.first() { + Some(first) => ( + first.description.clone().unwrap_or_default(), + first.unit.clone().unwrap_or_default(), + ), + None => (String::new(), String::new()), + }; + + // For counters, start_time is the boot time (when counting began). + // For gauges, start_time is not applicable (left unset). + let start_time = match metric_type { + MetricType::Counter => boot_time_nano, + MetricType::Gauge => 0, + }; + + // Duplicate attribute sets within one metric violate the OTLP single-writer + // principle and are rejected (or arbitrarily resolved) by backends. They are + // reachable via textfiles and duplicated mount entries, so drop repeats here. + let mut seen: HashSet> = HashSet::with_capacity(group.len()); + let mut dropped = 0usize; + let mut data_points: Vec = Vec::with_capacity(group.len()); + + for mut m in group { + // Compare label *sets*: `{a="1",b="2"}` and `{b="2",a="1"}` are the same + // OTLP identity, and textfile labels arrive in whatever order the file used. + // Sorting in place also makes the emitted attribute order stable across scrapes. + m.labels.sort_unstable(); + if !seen.insert(m.labels.clone()) { + dropped += 1; + continue; + } + + let attributes: Vec = m + .labels + .into_iter() + .map(|(k, v)| KeyValue { + key: k, + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(v)), + }), + + key_strindex: 0, + }) + .collect(); + + data_points.push(NumberDataPoint { + attributes, + start_time_unix_nano: start_time, + time_unix_nano, + exemplars: vec![], + flags: 0, + value: Some(number_data_point::Value::AsDouble(m.value)), + }); + } + + if dropped > 0 { + // Debug for the same reason: duplicated mounts or a bad `.prom` file do not + // self-heal, so this would repeat every scrape. + tracing::debug!( + "Dropped {} duplicate data point(s) for metric {}", + dropped, + name + ); + } + + let data = match metric_type { + MetricType::Gauge => { + opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(Gauge { data_points }) + } + MetricType::Counter => opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(Sum { + data_points, + aggregation_temporality: AggregationTemporality::Cumulative as i32, + is_monotonic: true, + }), + }; + + Metric { + name, + description, + unit, + metadata: vec![], + data: Some(data), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_test_metric( + name: &str, + value: f64, + metric_type: MetricType, + labels: Vec<(String, String)>, + ) -> CollectedMetric { + CollectedMetric { + name: name.to_string(), + value, + labels, + metric_type, + unit: Some("By".to_string()), + description: Some("Test metric".to_string()), + } + } + + #[test] + fn test_convert_gauge_metric() { + let metric = make_test_metric("node_load1", 1.5, MetricType::Gauge, vec![]); + + let resource_metrics = convert_to_otlp_metrics(vec![metric], 1234567890, "node_metrics"); + + assert!(resource_metrics.resource.is_some()); + assert_eq!(resource_metrics.scope_metrics.len(), 1); + assert_eq!(resource_metrics.scope_metrics[0].metrics.len(), 1); + + let otlp_metric = &resource_metrics.scope_metrics[0].metrics[0]; + assert_eq!(otlp_metric.name, "node_load1"); + + match &otlp_metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(gauge)) => { + assert_eq!(gauge.data_points.len(), 1); + match gauge.data_points[0].value { + Some(number_data_point::Value::AsDouble(v)) => { + assert!((v - 1.5).abs() < f64::EPSILON); + } + _ => panic!("Expected double value"), + } + } + _ => panic!("Expected Gauge data"), + } + } + + #[test] + fn test_convert_counter_metric() { + let metric = make_test_metric( + "node_network_receive_bytes_total", + 1000.0, + MetricType::Counter, + vec![], + ); + + let resource_metrics = convert_to_otlp_metrics(vec![metric], 1234567890, "node_metrics"); + + let otlp_metric = &resource_metrics.scope_metrics[0].metrics[0]; + assert_eq!(otlp_metric.name, "node_network_receive_bytes_total"); + + match &otlp_metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(sum)) => { + assert!(sum.is_monotonic); + assert_eq!( + sum.aggregation_temporality, + AggregationTemporality::Cumulative as i32 + ); + } + _ => panic!("Expected Sum data"), + } + } + + #[test] + fn test_convert_metric_with_labels() { + let labels = vec![("device".to_string(), "eth0".to_string())]; + + let metric = make_test_metric( + "node_network_receive_bytes_total", + 1000.0, + MetricType::Counter, + labels, + ); + + let resource_metrics = convert_to_otlp_metrics(vec![metric], 1234567890, "node_metrics"); + + let otlp_metric = &resource_metrics.scope_metrics[0].metrics[0]; + + match &otlp_metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(sum)) => { + assert_eq!(sum.data_points[0].attributes.len(), 1); + assert_eq!(sum.data_points[0].attributes[0].key, "device"); + // The value has to survive the conversion too: a label whose key is + // carried but whose value is dropped would still pass a key-only check. + assert_eq!( + attribute_pairs(&sum.data_points[0].attributes), + vec![("device", "eth0")] + ); + } + _ => panic!("Expected Sum data"), + } + } + + #[test] + fn test_convert_multiple_metrics_same_name_grouped() { + // Multiple data points for the same metric name should be grouped + let metrics = vec![ + make_test_metric( + "node_cpu_seconds_total", + 10.0, + MetricType::Counter, + vec![ + ("cpu".to_string(), "0".to_string()), + ("mode".to_string(), "user".to_string()), + ], + ), + make_test_metric( + "node_cpu_seconds_total", + 20.0, + MetricType::Counter, + vec![ + ("cpu".to_string(), "0".to_string()), + ("mode".to_string(), "system".to_string()), + ], + ), + ]; + + let resource_metrics = convert_to_otlp_metrics(metrics, 1234567890, "node_metrics"); + + // Should produce a single Metric with 2 data points + assert_eq!(resource_metrics.scope_metrics[0].metrics.len(), 1); + match &resource_metrics.scope_metrics[0].metrics[0].data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(sum)) => { + assert_eq!(sum.data_points.len(), 2); + // Which two points survive matters as much as how many: grouping that + // kept one point twice, or swapped a value onto the wrong attribute + // set, would still leave the count at 2. + assert_eq!( + point_summaries(&sum.data_points), + vec![ + (vec![("cpu", "0"), ("mode", "user")], 10.0), + (vec![("cpu", "0"), ("mode", "system")], 20.0), + ] + ); + } + _ => panic!("Expected Sum data"), + } + } + + #[test] + fn test_resource_has_service_name() { + let metric = make_test_metric("node_load1", 1.0, MetricType::Gauge, vec![]); + + let resource_metrics = + convert_to_otlp_metrics(vec![metric], 1234567890, "my_custom_service"); + + let resource = resource_metrics.resource.unwrap(); + + assert_eq!( + attribute_value(&resource, "service.name").as_deref(), + Some("my_custom_service") + ); + + // host.name must be present so series from different hosts stay distinguishable. + // The value itself is host-dependent and may be empty in a minimal container. + assert!( + resource.attributes.iter().any(|a| a.key == "host.name"), + "expected a host.name resource attribute, got {:?}", + resource.attributes + ); + + #[cfg(target_os = "linux")] + assert_eq!( + attribute_value(&resource, "os.type").as_deref(), + Some("linux") + ); + } + + /// The `(key, value)` pairs of a data point's attributes, in emitted order + fn attribute_pairs(attributes: &[KeyValue]) -> Vec<(&str, &str)> { + attributes + .iter() + .map(|kv| { + let value = match &kv.value { + Some(AnyValue { + value: Some(any_value::Value::StringValue(v)), + }) => v.as_str(), + other => panic!( + "expected a string attribute for {}, got {:?}", + kv.key, other + ), + }; + (kv.key.as_str(), value) + }) + .collect() + } + + /// The `(attributes, value)` of every data point, in emitted order + fn point_summaries(points: &[NumberDataPoint]) -> Vec<(Vec<(&str, &str)>, f64)> { + points + .iter() + .map(|p| { + let value = match p.value { + Some(number_data_point::Value::AsDouble(v)) => v, + other => panic!("expected a double data point value, got {:?}", other), + }; + (attribute_pairs(&p.attributes), value) + }) + .collect() + } + + /// The data points of the single metric in a converted batch + fn only_data_points(resource_metrics: &ResourceMetrics) -> &[NumberDataPoint] { + let metrics = &resource_metrics.scope_metrics[0].metrics; + assert_eq!(metrics.len(), 1, "expected exactly one metric"); + match &metrics[0].data { + Some(metric::Data::Gauge(gauge)) => &gauge.data_points, + Some(metric::Data::Sum(sum)) => &sum.data_points, + other => panic!("expected gauge or sum data, got {:?}", other), + } + } + + /// Look up a string-valued resource attribute by key + fn attribute_value(resource: &Resource, key: &str) -> Option { + resource + .attributes + .iter() + .find(|a| a.key == key) + .and_then(|a| match &a.value { + Some(AnyValue { + value: Some(any_value::Value::StringValue(v)), + }) => Some(v.clone()), + _ => None, + }) + } + + #[test] + fn test_same_name_gauge_and_counter_stay_separate() { + // Reachable through textfiles, whose `# TYPE` scope is per file: the two must not + // be merged into one metric with a single temporality. + let gauge = make_test_metric("collision", 1.0, MetricType::Gauge, vec![]); + let counter = make_test_metric("collision", 2.0, MetricType::Counter, vec![]); + + let resource_metrics = convert_to_otlp_metrics(vec![gauge, counter], 1000, "node_metrics"); + let metrics = &resource_metrics.scope_metrics[0].metrics; + + assert_eq!(metrics.len(), 2, "expected separate gauge and sum metrics"); + assert!(metrics.iter().all(|m| m.name == "collision")); + + let has_gauge = metrics.iter().any(|m| { + matches!( + m.data, + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(_)) + ) + }); + let sum = metrics.iter().find_map(|m| match &m.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(s)) => Some(s), + _ => None, + }); + + assert!(has_gauge, "gauge variant missing"); + let sum = sum.expect("sum variant missing"); + // The counter's points keep the boot-time start, the gauge's stay unset + assert_eq!( + sum.data_points[0].start_time_unix_nano, + 1000 * 1_000_000_000 + ); + } + + #[test] + fn test_duplicate_attribute_sets_are_dropped() { + // Two samples with identical label sets would violate the OTLP single-writer + // principle and can be rejected wholesale by backends. + let labels = vec![("device".to_string(), "sda".to_string())]; + let first = make_test_metric("node_disk_io_now", 1.0, MetricType::Gauge, labels.clone()); + let duplicate = make_test_metric("node_disk_io_now", 2.0, MetricType::Gauge, labels); + let other = make_test_metric( + "node_disk_io_now", + 3.0, + MetricType::Gauge, + vec![("device".to_string(), "sdb".to_string())], + ); + + let resource_metrics = + convert_to_otlp_metrics(vec![first, duplicate, other], 1000, "node_metrics"); + let metric = &resource_metrics.scope_metrics[0].metrics[0]; + + match &metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(gauge)) => { + assert_eq!(gauge.data_points.len(), 2); + // First occurrence wins, and the point that is *not* a duplicate keeps + // its own value: dropping `sdb` instead would also leave two points. + assert_eq!( + point_summaries(&gauge.data_points), + vec![ + (vec![("device", "sda")], 1.0), + (vec![("device", "sdb")], 3.0), + ] + ); + } + _ => panic!("Expected Gauge data"), + } + } + + #[test] + fn test_duplicate_attribute_sets_are_detected_regardless_of_label_order() { + // The same OTLP identity written with its labels in two different orders, as + // textfile lines and repeated mount entries can produce. Comparing the label + // vectors as given would treat these as two distinct series, so both would be + // emitted and the backend would reject or arbitrarily resolve them. + let unsorted = make_test_metric( + "node_disk_io_now", + 1.0, + MetricType::Gauge, + vec![ + ("b".to_string(), "2".to_string()), + ("a".to_string(), "1".to_string()), + ], + ); + let sorted = make_test_metric( + "node_disk_io_now", + 2.0, + MetricType::Gauge, + vec![ + ("a".to_string(), "1".to_string()), + ("b".to_string(), "2".to_string()), + ], + ); + + let resource_metrics = + convert_to_otlp_metrics(vec![unsorted, sorted], 1000, "node_metrics"); + let points = only_data_points(&resource_metrics); + + // One point survives, and its attributes are emitted in sorted order: the + // in-place sort that makes the comparison order-insensitive also normalises + // what is exported, so the attribute order is stable across scrapes. + assert_eq!( + point_summaries(points), + vec![(vec![("a", "1"), ("b", "2")], 1.0)] + ); + } + + #[test] + fn test_count_data_points_counts_exported_points_not_collected_ones() { + // A duplicate label set is dropped during conversion, so the collected count + // over-reports what was actually exported. + let labels = vec![("device".to_string(), "sda".to_string())]; + let collected = vec![ + make_test_metric("node_disk_io_now", 1.0, MetricType::Gauge, labels.clone()), + make_test_metric("node_disk_io_now", 2.0, MetricType::Gauge, labels), + make_test_metric( + "node_disk_io_now", + 3.0, + MetricType::Gauge, + vec![("device".to_string(), "sdb".to_string())], + ), + // A counter as well, so the Sum arm of the count is exercised too. + make_test_metric( + "node_disk_reads_completed_total", + 4.0, + MetricType::Counter, + vec![("device".to_string(), "sda".to_string())], + ), + ]; + let collected_count = collected.len(); + + let resource_metrics = convert_to_otlp_metrics(collected, 1000, "node_metrics"); + + // Two gauge points (the duplicate dropped) plus one counter point. + assert_eq!(count_data_points(&resource_metrics), 3); + assert_ne!( + count_data_points(&resource_metrics), + collected_count, + "the point count must reflect the conversion's drops" + ); + + // Cross-check against the emitted metrics rather than a hard-coded number. + let emitted: usize = resource_metrics.scope_metrics[0] + .metrics + .iter() + .map(|m| match &m.data { + Some(metric::Data::Gauge(g)) => g.data_points.len(), + Some(metric::Data::Sum(s)) => s.data_points.len(), + other => panic!("unexpected metric data {:?}", other), + }) + .sum(); + assert_eq!(count_data_points(&resource_metrics), emitted); + } + + #[test] + fn test_count_data_points_of_an_empty_batch_is_zero() { + let resource_metrics = convert_to_otlp_metrics(vec![], 1000, "node_metrics"); + assert_eq!(count_data_points(&resource_metrics), 0); + } + + #[test] + fn test_gauge_start_time_is_unset_and_metadata_propagates() { + let metric = make_test_metric("node_load1", 1.5, MetricType::Gauge, vec![]); + + let resource_metrics = convert_to_otlp_metrics(vec![metric], 1234567890, "node_metrics"); + let otlp_metric = &resource_metrics.scope_metrics[0].metrics[0]; + + // unit/description come from the collected metric + assert_eq!(otlp_metric.unit, "By"); + assert_eq!(otlp_metric.description, "Test metric"); + + let scope = resource_metrics.scope_metrics[0].scope.as_ref().unwrap(); + assert_eq!(scope.name, "node_metrics"); + + match &otlp_metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(gauge)) => { + assert_eq!(gauge.data_points[0].start_time_unix_nano, 0); + assert!(gauge.data_points[0].time_unix_nano > 0); + } + _ => panic!("Expected Gauge data"), + } + } + + #[test] + fn test_metrics_are_grouped_alphabetically() { + let metrics = vec![ + make_test_metric("node_c", 1.0, MetricType::Gauge, vec![]), + make_test_metric("node_a", 2.0, MetricType::Gauge, vec![]), + make_test_metric("node_b", 3.0, MetricType::Gauge, vec![]), + ]; + + let resource_metrics = convert_to_otlp_metrics(metrics, 1000, "node_metrics"); + let names: Vec<&str> = resource_metrics.scope_metrics[0] + .metrics + .iter() + .map(|m| m.name.as_str()) + .collect(); + + assert_eq!(names, vec!["node_a", "node_b", "node_c"]); + } +} diff --git a/src/receivers/node_metrics/mod.rs b/src/receivers/node_metrics/mod.rs new file mode 100644 index 00000000..ba76f7d1 --- /dev/null +++ b/src/receivers/node_metrics/mod.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Node metrics receiver +//! +//! Collects system metrics (CPU, memory, load, network, disk, filesystem, etc.) and +//! converts them to OpenTelemetry metrics. + +pub mod collector; +pub mod config; +pub mod convert; +pub mod receiver; + +pub use config::NodeMetricsConfig; +pub use receiver::NodeMetricsReceiver; diff --git a/src/receivers/node_metrics/receiver.rs b/src/receivers/node_metrics/receiver.rs new file mode 100644 index 00000000..feb5c58b --- /dev/null +++ b/src/receivers/node_metrics/receiver.rs @@ -0,0 +1,896 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Node metrics receiver implementation +//! +//! Periodically collects system metrics and sends them to the pipeline. + +use crate::receivers::get_meter; +use crate::receivers::node_metrics::collector::{CollectedMetric, SystemCollector}; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use crate::receivers::node_metrics::convert::{convert_to_otlp_metrics, count_data_points}; +use crate::receivers::otlp_output::OTLPOutput; +use crate::topology::payload; +use opentelemetry::KeyValue; +use opentelemetry::metrics::Counter; +use opentelemetry_proto::tonic::metrics::v1::ResourceMetrics; +use std::sync::Arc; +use tokio::task::{JoinHandle, JoinSet}; +use tokio_util::sync::CancellationToken; +use tower::BoxError; +use tracing::{debug, error, info, warn}; + +/// Node metrics receiver that periodically scrapes system metrics +pub struct NodeMetricsReceiver { + config: NodeMetricsConfig, + metrics_output: Option>>, +} + +impl NodeMetricsReceiver { + /// Create a new node metrics receiver + /// + /// The configuration is validated here, as the sibling receivers do, so a config built + /// programmatically rather than from the CLI cannot bypass the checks. + pub fn new( + mut config: NodeMetricsConfig, + metrics_output: Option>>, + ) -> Result { + config.normalize_and_validate()?; + Ok(Self { + config, + metrics_output, + }) + } + + /// Start the receiver, spawning a task that periodically collects metrics + pub fn start( + self, + task_set: &mut JoinSet>, + receivers_cancel: &CancellationToken, + ) -> Result<(), BoxError> { + let cancel = receivers_cancel.clone(); + let config = self.config; + let metrics_output = self.metrics_output; + + info!( + scrape_interval = ?config.scrape_interval, + collectors = ?config.collectors, + "Node metrics receiver starting" + ); + + #[cfg(not(target_os = "linux"))] + warn!( + "Node metrics receiver is designed for Linux. \ + Most collectors read from /proc and /sys which may not be available on this platform." + ); + + task_set.spawn(async move { + let result = run_scrape_loop(config, metrics_output, cancel).await; + if let Err(ref e) = result { + error!("Node metrics receiver error: {}", e); + } + result + }); + + Ok(()) + } +} + +/// How long a final batch may take to hand off once cancellation has been requested +const SHUTDOWN_SEND_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); + +/// Lower bound on how long a scrape may take before it is considered wedged +/// +/// The timeout is the scrape interval or this, whichever is longer: a short interval should +/// not turn a merely slow host into a receiver that never reports at all. +const MIN_SCRAPE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// How long to wait for the pipeline to accept a batch before dropping it +/// +/// An upper bound, further capped by the scrape interval at each use site, so a batch is +/// never held past the next tick and a long interval does not buy a long stall. +const SEND_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +/// Metrics counters for the receiver +struct ReceiverMetrics { + accepted: Counter, + refused: Counter, + scrape_failures: Counter, + empty_scrapes: Counter, + tags: [KeyValue; 1], +} + +impl ReceiverMetrics { + fn new() -> Self { + Self { + // Use the same accepted/refused vocabulary as the other receivers, so node + // metrics show up on the existing internal-telemetry dashboards. + // The description is part of the instrument identity, so it must match the + // other receivers declaring these names or the SDK reports a duplicate + // metric stream and exports two conflicting series. + accepted: get_meter() + .u64_counter("rotel_receiver_accepted_metric_points") + .with_description( + "Number of metric points successfully ingested and pushed into the pipeline.", + ) + .with_unit("metric_points") + .build(), + refused: get_meter() + .u64_counter("rotel_receiver_refused_metric_points") + .with_description( + "Number of metric points that could not be pushed into the pipeline.", + ) + .with_unit("metric_points") + .build(), + scrape_failures: get_meter() + .u64_counter("rotel_receiver_scrape_failures") + .with_description("Number of failed scrape attempts.") + .with_unit("scrapes") + .build(), + empty_scrapes: get_meter() + .u64_counter("rotel_receiver_empty_scrapes") + .with_description( + "Number of scrapes that collected no metrics at all, which usually means a \ + misconfigured procfs/sysfs path or an over-broad metric filter.", + ) + .with_unit("scrapes") + .build(), + tags: [KeyValue::new("receiver", "node_metrics")], + } + } + + fn add_accepted(&self, count: u64) { + if count > 0 { + self.accepted.add(count, &self.tags); + } + } + + fn add_refused(&self, count: u64) { + if count > 0 { + self.refused.add(count, &self.tags); + } + } + + fn add_scrape_failure(&self) { + self.scrape_failures.add(1, &self.tags); + } + + fn add_empty_scrape(&self) { + self.empty_scrapes.add(1, &self.tags); + } +} + +/// Decide whether a previously timed-out scrape is still occupying a blocking thread. +/// +/// Returns `true` when the earlier scrape has not finished, in which case this cycle must +/// be skipped: tokio cannot abort a blocking task, so starting another one would leak a +/// thread from the pool shared with the exporters every interval. A finished task is +/// reaped (surfacing a panic) and cleared, allowing scraping to resume. +async fn scrape_still_running(outstanding: &mut Option>>) -> bool { + let Some(handle) = outstanding else { + return false; + }; + + if !handle.is_finished() { + return true; + } + + if let Err(e) = handle.await { + warn!("Previously timed-out scrape task failed: {}", e); + } + *outstanding = None; + false +} + +/// Main scrape loop +async fn run_scrape_loop( + config: NodeMetricsConfig, + metrics_output: Option>>, + cancel: CancellationToken, +) -> Result<(), BoxError> { + let Some(output) = metrics_output else { + // Returning here would complete this receiver task, which the agent treats as an + // unexpected early exit and responds to by shutting everything down. Idle until + // cancellation instead. + warn!( + "No metrics output configured; node metrics receiver will not scrape. \ + Configure an exporter that accepts metrics." + ); + cancel.cancelled().await; + return Ok(()); + }; + + let receiver_metrics = ReceiverMetrics::new(); + let config = Arc::new(config); + + // Constructing the collector reads /proc/stat, so keep it off the async worker too. + let collector = { + let config = Arc::clone(&config); + Arc::new( + tokio::task::spawn_blocking(move || { + SystemCollector::new(&config.procfs_path, &config.sysfs_path) + }) + .await + .map_err(|e| format!("failed to initialize node metrics collector: {}", e))?, + ) + }; + + debug!( + "Node metrics scrape loop started, interval: {:?}, boot_time: {}", + config.scrape_interval, + collector.boot_time() + ); + + // Use interval_at to start immediately, then at regular intervals. + // Skip (not Delay or Burst) keeps ticks aligned to the original schedule after a slow + // scrape, instead of shifting the whole series forward or firing catch-up ticks. + let start = tokio::time::Instant::now(); + let mut interval = tokio::time::interval_at(start, config.scrape_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + // A scrape that timed out keeps running: tokio cannot abort a blocking task once it + // has started. Hold on to it so the loop does not start a fresh scrape (and leak + // another blocking thread) every interval while the previous one is wedged — that + // would eventually exhaust the blocking pool shared with the exporters. + let mut outstanding: Option>> = None; + + loop { + tokio::select! { + biased; + + _ = cancel.cancelled() => { + info!("Node metrics receiver cancelled"); + break; + } + + _ = interval.tick() => {} + } + + if scrape_still_running(&mut outstanding).await { + warn!( + "Previous scrape is still running or queued, skipping this cycle. \ + If this persists, a filesystem or sysfs read is likely unresponsive; \ + a filesystem can be excluded with \ + --node-metrics-receiver-filesystem-mount-exclude." + ); + receiver_metrics.add_scrape_failure(); + continue; + } + + debug!("Scraping node metrics"); + + // Collect metrics on a blocking thread to avoid stalling the async runtime + // (statfs and /proc reads can block). The timeout bounds the loop, not the + // blocking task itself, and cancellation is honoured while waiting. + // Never shorter than MIN_SCRAPE_TIMEOUT: with the 1s interval the CLI permits, a + // scrape that legitimately takes longer would time out on every cycle and the + // receiver would emit nothing at all rather than merely lagging. + let scrape_timeout = config.scrape_interval.max(MIN_SCRAPE_TIMEOUT); + let mut handle = { + let collector = Arc::clone(&collector); + let config = Arc::clone(&config); + tokio::task::spawn_blocking(move || collector.collect(&config)) + }; + + let collected = tokio::select! { + biased; + + _ = cancel.cancelled() => { + info!("Node metrics receiver cancelled during scrape"); + break; + } + + res = tokio::time::timeout(scrape_timeout, &mut handle) => match res { + Ok(Ok(collected)) => collected, + Ok(Err(e)) => { + // Do not return Err: the agent propagates a receiver error *before* + // cancelling receivers and flushing exporters, so one panicking + // collector would discard unrelated pipelines' buffered telemetry. + // Treat it like the abandoned-task case and keep scraping. + warn!("Node metrics scrape task failed: {}", e); + receiver_metrics.add_scrape_failure(); + continue; + } + Err(_) => { + warn!("Scrape timed out after {:?}, skipping this cycle", scrape_timeout); + receiver_metrics.add_scrape_failure(); + // Keep the handle so the next tick can tell whether the blocking task + // is still wedged, instead of spawning another one on top of it. + outstanding = Some(handle); + continue; + } + } + }; + + let metric_count = collected.len(); + + if metric_count == 0 { + // Not debug-only: an empty scrape every interval means a wrong procfs/sysfs + // path or an over-broad exclude filter, which is otherwise invisible. + // Counted as well as logged: a permanently empty scrape is otherwise + // indistinguishable from a receiver that was never enabled. + warn!( + "No metrics collected; check the configured procfs/sysfs paths and metric filters" + ); + receiver_metrics.add_empty_scrape(); + continue; + } + + // Convert to OTLP format. Counters carry the *latched* boot time, so that a wall + // clock step does not move their start time and read as a counter reset. + let resource_metrics = convert_to_otlp_metrics( + collected, + collector.counter_start_time(), + &config.service_name, + ); + + // Count the points actually exported, not the ones collected: the conversion + // drops duplicate label sets. + let point_count = count_data_points(&resource_metrics); + + // Send to an output channel + let message = payload::Message::new(None, vec![resource_metrics], None); + + // Hold the send future so cancellation can give the in-flight send a bounded + // window to finish rather than discarding the batch outright. + let mut send_fut = std::pin::pin!(output.send_async(message)); + + tokio::select! { + biased; + + _ = cancel.cancelled() => { + // The pipeline is still draining at this point, so let the send finish if + // it can. The window stays well inside the agent's receiver drain budget. + match tokio::time::timeout(SHUTDOWN_SEND_TIMEOUT, send_fut.as_mut()).await { + Ok(Ok(())) => { + debug!("Sent {} metric points during shutdown", point_count); + receiver_metrics.add_accepted(point_count as u64); + } + Ok(Err(e)) => { + info!( + "Metrics pipeline already closed during shutdown, dropping {} \ + metric points: {}", + point_count, e + ); + receiver_metrics.add_refused(point_count as u64); + } + Err(_) => { + info!( + "Metrics pipeline did not accept {} metric points within {:?} of \ + shutdown, dropping them", + point_count, SHUTDOWN_SEND_TIMEOUT + ); + receiver_metrics.add_refused(point_count as u64); + } + } + break; + } + + // Bound the send: a full channel (slow or retrying exporter) would otherwise + // park here indefinitely, silently stopping all scraping. + res = tokio::time::timeout( + SEND_TIMEOUT.min(config.scrape_interval), + send_fut.as_mut(), + ) => match res { + Ok(Ok(())) => { + debug!("Sent {} metric points to pipeline", point_count); + receiver_metrics.add_accepted(point_count as u64); + } + Ok(Err(e)) => { + // The only send error is a disconnected pipeline. During shutdown that + // is expected, so do not report it as a receiver failure; otherwise + // retrying forever would hide it from the agent. + receiver_metrics.add_refused(point_count as u64); + if cancel.is_cancelled() { + info!("Metrics pipeline closed during shutdown: {}", e); + break; + } + error!("Failed to send metrics to pipeline: {}", e); + return Err(format!("node metrics pipeline disconnected: {}", e).into()); + } + Err(_) => { + // Drop this batch rather than stalling: the next scrape re-reads the + // current values, so a dropped batch is recoverable. + warn!( + "Metrics pipeline did not accept {} metric points within {:?}, \ + abandoning this batch (it may still be delivered). The metrics \ + exporter is likely backed up.", + point_count, + SEND_TIMEOUT.min(config.scrape_interval) + ); + receiver_metrics.add_refused(point_count as u64); + } + } + } + } + + info!("Node metrics receiver stopped"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bounded_channel::bounded; + use std::time::Duration; + use tracing_test::traced_test; + + use crate::receivers::node_metrics::config::Collector; + + /// Count captured log lines containing `needle`. + /// + /// `logs_assert` is the only accessor `#[traced_test]` provides, so the count is + /// captured out of the closure. + macro_rules! logs_with { + ($needle:expr) => {{ + let count = std::cell::Cell::new(0usize); + logs_assert(|lines: &[&str]| { + count.set(lines.iter().filter(|line| line.contains($needle)).count()); + Ok(()) + }); + count.get() + }}; + } + + /// Drive the loop until it has logged the start of a scrape, then give it a short grace + /// period to reach the send. Waiting on the log rather than on a fixed duration keeps + /// the test deterministic on a loaded machine, where scheduling a blocking task can + /// take far longer than a hard-coded sleep allows. + macro_rules! drive_until_scraping { + ($loop_fut:expr) => {{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !logs_contain("Scraping node metrics") { + assert!( + tokio::time::Instant::now() < deadline, + "the scrape loop never started a scrape" + ); + tokio::select! { + res = $loop_fut.as_mut() => panic!("scrape loop exited early: {:?}", res), + _ = tokio::time::sleep(Duration::from_millis(10)) => {} + } + } + // The scrape itself only reads the clock and calls uname, so this is ample. + tokio::select! { + res = $loop_fut.as_mut() => panic!("scrape loop exited early: {:?}", res), + _ = tokio::time::sleep(Duration::from_millis(100)) => {} + } + }}; + } + + /// A config that scrapes only cheap, always-available collectors + fn test_config() -> NodeMetricsConfig { + NodeMetricsConfig { + scrape_interval: Duration::from_secs(1), + collectors: vec![Collector::Uname, Collector::Time], + ..Default::default() + } + } + + /// [`test_config`] with a different scrape interval + /// + /// The interval also bounds the send (`SEND_TIMEOUT.min(scrape_interval)`), so it is + /// what makes the backpressure tests finish quickly. + fn test_config_with_interval(scrape_interval: Duration) -> NodeMetricsConfig { + NodeMetricsConfig { + scrape_interval, + ..test_config() + } + } + + /// A placeholder batch, used to occupy channel capacity + fn placeholder_message() -> payload::Message { + payload::Message::new(None, vec![], None) + } + + #[tokio::test] + async fn test_scrape_loop_cancellation() { + let (tx, _rx) = bounded::>(10); + let output = OTLPOutput::new(tx); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + + let handle = tokio::spawn(async move { + run_scrape_loop(test_config(), Some(output), cancel_clone).await + }); + + // Let it run briefly + tokio::time::sleep(Duration::from_millis(100)).await; + + cancel.cancel(); + + // Unwrap both layers: the join result and the loop's own Result, so a panicking + // or failing loop cannot pass this test. + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop task panicked") + .expect("scrape loop returned an error"); + } + + /// With no metrics pipeline, the loop must idle until cancelled rather than returning. + /// Returning early is what the agent interprets as an unexpected receiver exit, which + /// makes it shut the whole process down. + #[tokio::test] + async fn test_no_metrics_output_idles_until_cancelled() { + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + + let mut handle = + tokio::spawn(async move { run_scrape_loop(test_config(), None, cancel_clone).await }); + + // It must still be running after a moment + tokio::select! { + res = &mut handle => panic!("scrape loop exited before cancellation: {:?}", res), + _ = tokio::time::sleep(Duration::from_millis(200)) => {} + } + + cancel.cancel(); + + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop task panicked") + .expect("scrape loop returned an error"); + } + + /// A disconnected pipeline must surface as an error instead of being retried forever. + #[tokio::test] + async fn test_disconnected_pipeline_returns_error() { + let (tx, rx) = bounded::>(1); + let output = OTLPOutput::new(tx); + // Drop the receiving end so the first send fails + drop(rx); + + let cancel = CancellationToken::new(); + let result = tokio::time::timeout( + Duration::from_secs(5), + run_scrape_loop(test_config(), Some(output), cancel), + ) + .await + .expect("scrape loop did not return after the pipeline disconnected"); + + let err = result.expect_err("expected an error when the pipeline is disconnected"); + assert!( + err.to_string().contains("pipeline disconnected"), + "expected a disconnect error, got: {}", + err + ); + } + + /// The latch that stops a wedged scrape from leaking a blocking thread every + /// interval. Driving it directly avoids needing an unresponsive filesystem. + #[tokio::test] + async fn test_wedged_scrape_skips_cycles_until_it_finishes() { + // Nothing outstanding: scraping proceeds. + let mut outstanding: Option>> = None; + assert!(!scrape_still_running(&mut outstanding).await); + + // A task that never finishes stands in for a hung procfs read. + let (release, wedged) = tokio::sync::oneshot::channel::<()>(); + let mut outstanding = Some(tokio::spawn(async move { + let _ = wedged.await; + Vec::new() + })); + + // Every cycle while it runs must be skipped, and the handle retained. + for _ in 0..3 { + assert!( + scrape_still_running(&mut outstanding).await, + "a running scrape must cause the cycle to be skipped" + ); + assert!(outstanding.is_some(), "the handle must be retained"); + } + + // Once it completes, the latch clears and scraping resumes. + release.send(()).unwrap(); + let mut cleared = false; + for _ in 0..100 { + if !scrape_still_running(&mut outstanding).await { + cleared = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(cleared, "the latch never cleared after the scrape finished"); + assert!(outstanding.is_none(), "the finished handle must be reaped"); + } + + /// A panic in an abandoned scrape must be surfaced, not discarded with the handle. + #[traced_test] + #[tokio::test] + async fn test_wedged_scrape_panic_is_reaped() { + let handle = tokio::spawn(async { panic!("boom") }); + // Let it finish so the latch takes the reaping path + let _ = tokio::time::timeout(Duration::from_secs(5), async { + while !handle.is_finished() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + + let mut outstanding = Some(handle); + assert!(!scrape_still_running(&mut outstanding).await); + assert!(outstanding.is_none()); + + // Reaping is the only place the panic can be observed: the handle is dropped + // immediately afterwards, so a silent reap would lose it entirely. + assert!( + logs_contain("Previously timed-out scrape task failed"), + "the reaped panic must be logged" + ); + } + + /// A full pipeline must cost one batch, not the scrape loop. + /// + /// The send is bounded so that a backed-up exporter cannot park the loop forever: + /// the batch is dropped and the next scrape re-reads the current values. + #[traced_test] + #[tokio::test] + async fn test_full_pipeline_abandons_the_batch_and_keeps_scraping() { + // Capacity 1, pre-filled and never drained, so every send the loop attempts + // parks. Holding `_rx` keeps the channel connected: a dropped receiver would + // take the disconnect path instead. + let (tx, mut rx) = bounded::>(1); + tx.send(placeholder_message()).await.unwrap(); + let output = OTLPOutput::new(tx.clone()); + + // The send is bounded by min(SEND_TIMEOUT, scrape_interval), so the interval is + // also how long the first batch waits before being abandoned. + let interval = Duration::from_millis(300); + let cancel = CancellationToken::new(); + let mut loop_fut = std::pin::pin!(run_scrape_loop( + test_config_with_interval(interval), + Some(output), + cancel.clone() + )); + + // Drive until two batches have been abandoned, rather than timing a fixed window: + // the assertion below is about the count, so wait for the count. + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + loop { + let abandoned = logs_with!("abandoning this batch"); + if abandoned >= 2 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "only {} batches were abandoned before the deadline", + abandoned + ); + tokio::select! { + res = loop_fut.as_mut() => { + panic!("scrape loop exited while the pipeline was full: {:?}", res) + } + _ = tokio::time::sleep(Duration::from_millis(20)) => {} + } + } + + // Warned, not silently dropped: this is the only signal that the exporter is + // backed up badly enough to be losing data. + assert!( + logs_contain("abandoning this batch"), + "an abandoned batch must be reported" + ); + + // More than one batch was abandoned, so the loop went on scraping instead of + // parking on the full channel for good after the first. + logs_assert(|lines: &[&str]| { + let abandoned = lines + .iter() + .filter(|line| line.contains("abandoning this batch")) + .count(); + if abandoned >= 2 { + Ok(()) + } else { + Err(format!( + "expected at least 2 abandoned batches, saw {}", + abandoned + )) + } + }); + + // The batch really was dropped rather than queued behind the placeholder. + // Draining shows the abandoned batches were never queued: only the placeholder is + // there. (Asserting `tx.len() == 1` would be vacuous — capacity is 1.) + let queued = rx.next().await.expect("channel closed"); + assert!( + queued.payload.is_empty(), + "only the placeholder should have been queued, got {} payload(s)", + queued.payload.len() + ); + + // The loop is still alive and still returns cleanly. + cancel.cancel(); + tokio::time::timeout(Duration::from_secs(5), loop_fut) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop returned an error"); + } + + /// Cancellation gives an in-flight send a bounded window to complete. + /// + /// The pipeline is still draining when receivers are cancelled, so a batch that can + /// still be handed off should be, rather than discarded outright. + #[traced_test] + #[tokio::test] + async fn test_shutdown_grace_window_lets_an_inflight_send_finish() { + // Pre-fill the single slot so the loop's first send parks, and use an interval + // long enough that it is certainly still parked when cancellation arrives. + let (tx, mut rx) = bounded::>(1); + tx.send(placeholder_message()).await.unwrap(); + let output = OTLPOutput::new(tx.clone()); + + let cancel = CancellationToken::new(); + let mut loop_fut = std::pin::pin!(run_scrape_loop( + test_config_with_interval(Duration::from_secs(30)), + Some(output), + cancel.clone() + )); + + // Let the first scrape finish and park in the send. + drive_until_scraping!(loop_fut); + + cancel.cancel(); + + // Freeing the slot lets the parked send complete inside SHUTDOWN_SEND_TIMEOUT. + let placeholder = rx.next().await.expect("channel closed"); + assert!( + placeholder.payload.is_empty(), + "expected the pre-filled placeholder to be drained first" + ); + + // The loop must return well inside the agent's 3 second receiver drain budget. + let started = tokio::time::Instant::now(); + tokio::time::timeout(Duration::from_secs(3), loop_fut) + .await + .expect("scrape loop did not stop within the receiver drain budget") + .expect("scrape loop returned an error"); + let elapsed = started.elapsed(); + + // The batch was handed off during the grace window, not dropped. + assert!( + logs_contain("metric points during shutdown"), + "the final batch should have been sent during shutdown" + ); + let batch = tokio::time::timeout(Duration::from_secs(1), rx.next()) + .await + .expect("timed out waiting for the final batch") + .expect("channel closed without the final batch"); + assert_eq!(batch.payload.len(), 1); + assert!( + !batch.payload[0].scope_metrics[0].metrics.is_empty(), + "the final batch must carry the scraped metrics" + ); + + // Completing early is the point: the window is a bound, not a delay. + assert!( + elapsed < SHUTDOWN_SEND_TIMEOUT, + "shutdown took {:?}, which means the send did not complete in the window", + elapsed + ); + } + + /// A scrape whose metrics are all filtered out must not send an empty batch. + #[traced_test] + #[tokio::test] + async fn test_fully_filtered_scrape_sends_nothing() { + let config = NodeMetricsConfig { + scrape_interval: Duration::from_secs(1), + collectors: vec![Collector::Uname, Collector::Time], + ..Default::default() + } + .with_exclude_filter(".*") + .expect("valid regex"); + + let (tx, mut rx) = bounded::>(10); + let output = OTLPOutput::new(tx); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let handle = + tokio::spawn(async move { run_scrape_loop(config, Some(output), cancel_clone).await }); + + // Nothing should arrive on the channel + let got = tokio::time::timeout(Duration::from_millis(500), rx.next()).await; + assert!( + got.is_err(), + "expected no message for a fully filtered scrape" + ); + + // ...and the absence of a message must be because the loop filtered everything + // out, not because it died on its first iteration. + let mut handle = handle; + tokio::select! { + res = &mut handle => panic!("scrape loop exited unexpectedly: {:?}", res), + _ = tokio::time::sleep(Duration::from_millis(50)) => {} + } + + // An empty scrape means a wrong procfs/sysfs path or an over-broad filter, and + // is otherwise invisible: nothing arrives on the channel either way. + assert!( + logs_contain("No metrics collected"), + "an empty scrape must be reported" + ); + + cancel.cancel(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop task panicked") + .expect("scrape loop returned an error"); + } + + /// Integration test: run a full scrape→convert→send cycle and verify + /// that ResourceMetrics actually arrive on the output channel. + /// Uses only the Uname collector which works on all unix platforms + /// (libc::uname + SystemTime::now). + #[tokio::test] + async fn test_scrape_produces_metrics_on_channel() { + let config = NodeMetricsConfig { + scrape_interval: Duration::from_secs(1), + collectors: vec![Collector::Uname, Collector::Time], + ..Default::default() + }; + + let (tx, mut rx) = bounded::>(10); + let output = OTLPOutput::new(tx); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + + let handle = + tokio::spawn(async move { run_scrape_loop(config, Some(output), cancel_clone).await }); + + // Wait for the first scrape to produce a message + let msg = tokio::time::timeout(Duration::from_secs(5), rx.next()) + .await + .expect("timed out waiting for metrics") + .expect("channel closed without receiving metrics"); + + cancel.cancel(); + + // Unwrap both layers, like the sibling tests: a loop that produced one good + // batch and then failed must not pass. + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop task panicked") + .expect("scrape loop returned an error"); + + // Verify the message contains ResourceMetrics with actual data + assert_eq!(msg.payload.len(), 1); + let resource_metrics = &msg.payload[0]; + + // Should have a resource with service.name + let resource = resource_metrics + .resource + .as_ref() + .expect("missing resource"); + assert!( + resource + .attributes + .iter() + .any(|kv| kv.key == "service.name") + ); + + // Should have node_uname_info (from Uname) and node_time_seconds (from Time) + let scope_metrics = &resource_metrics.scope_metrics; + assert_eq!(scope_metrics.len(), 1); + let metric_names: Vec<&str> = scope_metrics[0] + .metrics + .iter() + .map(|m| m.name.as_str()) + .collect(); + for expected in ["node_time_seconds", "node_uname_info"] { + assert!( + metric_names.contains(&expected), + "expected {} in {:?}", + expected, + metric_names + ); + } + + // host.name must be on the resource so multi-host series stay distinguishable + assert!(resource.attributes.iter().any(|kv| kv.key == "host.name")); + } +} diff --git a/tests/node_metrics_integration_tests.rs b/tests/node_metrics_integration_tests.rs new file mode 100644 index 00000000..12f8b90b --- /dev/null +++ b/tests/node_metrics_integration_tests.rs @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Node Metrics Integration Tests +//! +//! These tests run against the real `/proc` and `/sys` of a Linux host. They exist to +//! catch what the fixture-based unit tests cannot: a drift between the kernel's actual +//! file formats and the parsers in this receiver. +//! +//! To run these tests: +//! +//! ``` +//! NODE_METRICS_INTEGRATION_TESTS=true cargo test --test node_metrics_integration_tests --features node_metrics_receiver +//! ``` +//! +//! Note: collectors for optional hardware (thermal zones, hwmon sensors, NVMe devices, +//! cpufreq, diskstats) are absent in many containers and virtual machines. For those the +//! tests accept an empty result and require only that any value produced is finite; they +//! never require a particular piece of hardware to be present, which would be flaky by +//! design. + +#![cfg(all( + target_os = "linux", + node_metrics_integration_tests = "true", + feature = "node_metrics_receiver" +))] + +use rotel::receivers::node_metrics::collector::{MetricType, SystemCollector}; +use rotel::receivers::node_metrics::config::{Collector, NodeMetricsConfig}; + +/// Build a config restricted to the given collectors, reading the real procfs/sysfs +fn config_for(collectors: Vec) -> NodeMetricsConfig { + NodeMetricsConfig { + collectors, + ..Default::default() + } +} + +#[test] +fn test_boot_time_is_plausible() { + let collector = SystemCollector::default(); + let boot_time = collector.boot_time(); + + // Deliberately tolerant: a board with no RTC reports a near-epoch btime until NTP + // syncs, and the collector itself treats 0 as "unknown" and suppresses the metric. So + // only require that the value is not in the future. + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + assert!( + boot_time <= now, + "boot time {} is in the future (now {})", + boot_time, + now + ); +} + +#[test] +fn test_core_collectors_produce_metrics_on_a_real_host() { + let collector = SystemCollector::default(); + let config = config_for(vec![ + Collector::Cpu, + Collector::Loadavg, + Collector::Memory, + Collector::Filesystem, + Collector::Stat, + Collector::Vmstat, + ]); + + let metrics = collector.collect(&config); + assert!( + !metrics.is_empty(), + "expected metrics from a real Linux host" + ); + + // These four exist on every Linux kernel this receiver targets. If the kernel ever + // changes a format the parser depends on, one of them disappears. + // node_filesystem_size_bytes is deliberately absent from this list: a container whose + // only mounts are overlay/proc/sysfs reports no filesystems at all, and those types are + // excluded by design. test_filesystem_stats_are_self_consistent covers it where present. + for expected in [ + "node_cpu_seconds_total", + "node_memory_MemTotal_bytes", + "node_load1", + ] { + assert!( + metrics.iter().any(|m| m.name == expected), + "missing {} from a real host scrape", + expected + ); + } + + // The payoff of running against a real kernel: no value is ever NaN or infinite, + // whatever the host looks like. + for m in &metrics { + assert!( + m.value.is_finite(), + "metric {} has a non-finite value: {}", + m.name, + m.value + ); + // Non-negativity is only asserted where it is part of the metric's definition — + // byte, second and filesystem gauges. Kernel-supplied fields in general (and + // temperatures in particular) may legitimately be negative. + let must_be_non_negative = m.name.ends_with("_bytes") + || m.name.ends_with("_bytes_total") + || m.name.ends_with("_seconds_total") + || m.name.starts_with("node_filesystem_") + || m.name.starts_with("node_load"); + if must_be_non_negative { + assert!( + m.value >= 0.0, + "metric {} has a negative value: {}", + m.name, + m.value + ); + } + } +} + +#[test] +fn test_filesystem_stats_are_self_consistent() { + let collector = SystemCollector::default(); + let metrics = collector.collect(&config_for(vec![Collector::Filesystem])); + + // Do not require a mount point of "/": inside a container the root is usually an + // overlay mount, which is excluded by filesystem type. Pick the largest reported + // filesystem instead, which exists on any host that reports one at all. + let Some(labels) = metrics + .iter() + .filter(|m| m.name == "node_filesystem_size_bytes") + .max_by(|a, b| a.value.total_cmp(&b.value)) + .map(|m| m.labels.clone()) + else { + eprintln!("Skipping test: no filesystems reported on this host"); + return; + }; + let mountpoint = labels + .iter() + .find(|(k, _)| k == "mountpoint") + .map(|(_, v)| v.as_str()) + .unwrap_or(""); + + // Key on the whole label set, not just the mount point: with an over-mount two + // devices share a mount point, and mixing their values would fail spuriously. + let value_for = |name: &str| -> Option { + metrics + .iter() + .find(|m| m.name == name && m.labels == labels) + .map(|m| m.value) + }; + + let size = value_for("node_filesystem_size_bytes").expect("no size"); + let free = value_for("node_filesystem_free_bytes").expect("no free"); + let avail = value_for("node_filesystem_avail_bytes").expect("no avail"); + + assert!( + size > 0.0, + "filesystem {} size should be positive", + mountpoint + ); + assert!( + avail <= free && free <= size, + "{}: expected avail ({}) <= free ({}) <= size ({})", + mountpoint, + avail, + free, + size + ); +} + +#[test] +fn test_counters_and_gauges_are_classified_on_a_real_host() { + let collector = SystemCollector::default(); + let metrics = collector.collect(&config_for(vec![Collector::Cpu, Collector::Stat])); + + let cpu = metrics + .iter() + .find(|m| m.name == "node_cpu_seconds_total") + .expect("no CPU metric"); + assert_eq!(cpu.metric_type, MetricType::Counter); + + let running = metrics + .iter() + .find(|m| m.name == "node_procs_running") + .expect("no node_procs_running metric"); + assert_eq!(running.metric_type, MetricType::Gauge); +} + +/// Collectors backed by interfaces every Linux kernel provides must return something. +/// If one silently stops parsing, this is what notices. +#[test] +fn test_always_available_collectors_produce_metrics() { + let collector = SystemCollector::default(); + + // Netstat, Sockstat and Filefd are deliberately excluded: they need + // /proc/net/{netstat,snmp,sockstat} and /proc/sys/fs/file-nr, which sandboxes such as + // gVisor and WSL1 only partially synthesise. They are covered by the tolerate-empty + // test below instead. + for c in [ + Collector::Network, + Collector::Uname, + Collector::Time, + Collector::Processes, + ] { + let metrics = collector.collect(&config_for(vec![c.clone()])); + assert!( + !metrics.is_empty(), + "collector {:?} produced no metrics on a real Linux host", + c + ); + for m in &metrics { + assert!( + m.value.is_finite(), + "collector {:?} produced a non-finite value for {}", + c, + m.name + ); + } + } +} + +/// Hardware-dependent collectors may legitimately produce nothing (container, VM, no +/// sensors), so assert only that they degrade quietly. Note these can report negative +/// values — sub-zero temperatures are legal — so no sign assertion here. +#[test] +fn test_optional_hardware_collectors_do_not_panic() { + let collector = SystemCollector::default(); + + for c in [ + Collector::Cpufreq, + Collector::ThermalZone, + Collector::Nvme, + Collector::Hwmon, + Collector::Diskstats, + // Present on ordinary kernels but only partially synthesised by some sandboxes + Collector::Netstat, + Collector::Sockstat, + Collector::Filefd, + ] { + let metrics = collector.collect(&config_for(vec![c.clone()])); + for m in &metrics { + assert!( + m.value.is_finite(), + "collector {:?} produced a non-finite value for {}", + c, + m.name + ); + } + } +} + +/// The invariant a fixture can never prove: on a real host's mount table, hwmon set and +/// network devices, no OTLP metric may contain two data points with the same attribute +/// set, and no (name, type) pair may appear twice. +#[test] +fn test_real_host_scrape_has_no_conflicting_data_points() { + use rotel::receivers::node_metrics::convert::convert_to_otlp_metrics; + + let collector = SystemCollector::default(); + let metrics = collector.collect(&NodeMetricsConfig::default()); + let resource_metrics = convert_to_otlp_metrics(metrics, collector.boot_time(), "node_metrics"); + + let mut seen_metrics = std::collections::HashSet::new(); + for sm in &resource_metrics.scope_metrics { + for m in &sm.metrics { + let points = match &m.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(g)) => { + assert!( + seen_metrics.insert((m.name.clone(), "gauge")), + "duplicate gauge {}", + m.name + ); + &g.data_points + } + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(s)) => { + assert!( + seen_metrics.insert((m.name.clone(), "sum")), + "duplicate sum {}", + m.name + ); + &s.data_points + } + _ => continue, + }; + + let mut seen_points = std::collections::HashSet::new(); + for dp in points { + let mut key: Vec<(String, String)> = dp + .attributes + .iter() + .map(|kv| { + let value = match &kv.value { + Some(v) => format!("{:?}", v.value), + None => String::new(), + }; + (kv.key.clone(), value) + }) + .collect(); + key.sort(); + assert!( + seen_points.insert(key.clone()), + "metric {} has duplicate data points for attributes {:?}", + m.name, + key + ); + } + } + } +} + +#[test] +fn test_hwmon_chip_labels_are_unique_per_sensor() { + // Two chips of the same model (e.g. two NVMe drives) must not collapse onto the same + // chip label, which would produce duplicate OTLP attribute sets. + let collector = SystemCollector::default(); + let metrics = collector.collect(&config_for(vec![Collector::Hwmon])); + + let mut seen = std::collections::HashSet::new(); + for m in metrics.iter().filter(|m| m.name != "node_hwmon_chip_names") { + let key = (m.name.clone(), m.labels.clone()); + assert!( + seen.insert(key), + "duplicate label set for {}: {:?}", + m.name, + m.labels + ); + } +}