From ebb0ea6db1bb59f51a365ec0ba53cf1a81e4e410 Mon Sep 17 00:00:00 2001 From: Neeme Praks Date: Fri, 20 Feb 2026 20:49:50 +0200 Subject: [PATCH] Add node metrics receiver for host-level system metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a new opt-in receiver (`--features node_metrics_receiver`) that periodically scrapes system metrics from Linux /proc and /sys interfaces and emits them as OpenTelemetry metrics using Prometheus node_exporter naming conventions. 19 collectors, 18 of them enabled by default: - CPU, load average, memory, network, filesystem, uname, time - Kernel stat counters, processes, disk I/O, vmstat - Network statistics (netstat + SNMP), socket statistics - File descriptor stats, CPU frequency, thermal zones and cooling devices - NVMe device info, hardware monitoring sensors (hwmon) - Textfile collector for custom Prometheus-format .prom files (opt-in) The scrape runs on a blocking thread and races against cancellation, so a slow or wedged filesystem cannot stall the runtime or overrun the agent's shutdown budget. Because tokio cannot abort a blocking task, a timed-out scrape is retained and not re-issued until it completes, which keeps a hung statfs from leaking a blocking thread every interval into the pool shared with the exporters. Network filesystem types are excluded by default for the same reason, while local FUSE mounts keep reporting. The pipeline send is bounded, so a backed-up exporter drops a batch rather than silently stopping all scraping, and a panicking collector is logged and counted rather than taking the agent down with it. Metrics are grouped by name and type, with duplicate label sets dropped, so a single OTLP metric never carries conflicting data points. Boot time is re-read every scrape, because the kernel's btime moves when the wall clock is stepped — a device without an RTC boots near the epoch and jumps when NTP syncs — while the start time stamped on cumulative counters stays latched so a clock step is not reported as a counter reset. The receiver's flags follow the `---receiver-*` convention used by the other receivers, durations are parsed with humantime like the rest of the CLI, and its internal telemetry reuses the shared rotel_receiver_accepted/refused_metric_points counters. Metric names, help strings, the default filesystem exclusion lists, the hwmon chip-naming approach and the textfile collector's semantics are derived from Prometheus node_exporter (Apache-2.0); this is recorded in the collector module and in each derived submodule. The CI test job now enables this feature, and the 32-bit ARM job builds it, since libc's statfs field widths differ there and AtomicU64 needs portable-atomic on targets without native 64-bit atomics. Without that, none of this code is compiled on a pull request and feature-gated code silently rots against dependency changes. --- .github/workflows/ci.yml | 8 +- Cargo.toml | 1 + README.md | 493 +++- build.rs | 5 + src/init/activation.rs | 75 + src/init/agent.rs | 13 + src/init/args.rs | 13 + src/init/config.rs | 8 + src/init/mod.rs | 3 + src/init/node_metrics_receiver.rs | 875 ++++++ src/receivers/mod.rs | 3 + src/receivers/node_metrics/collector/mod.rs | 478 +++ .../node_metrics/collector/procfs.rs | 2575 +++++++++++++++++ src/receivers/node_metrics/collector/sysfs.rs | 991 +++++++ .../node_metrics/collector/test_support.rs | 149 + .../node_metrics/collector/textfile.rs | 1223 ++++++++ src/receivers/node_metrics/collector/util.rs | 370 +++ src/receivers/node_metrics/config.rs | 323 +++ src/receivers/node_metrics/convert.rs | 684 +++++ src/receivers/node_metrics/mod.rs | 14 + src/receivers/node_metrics/receiver.rs | 896 ++++++ tests/node_metrics_integration_tests.rs | 329 +++ 22 files changed, 9521 insertions(+), 8 deletions(-) create mode 100644 src/init/node_metrics_receiver.rs create mode 100644 src/receivers/node_metrics/collector/mod.rs create mode 100644 src/receivers/node_metrics/collector/procfs.rs create mode 100644 src/receivers/node_metrics/collector/sysfs.rs create mode 100644 src/receivers/node_metrics/collector/test_support.rs create mode 100644 src/receivers/node_metrics/collector/textfile.rs create mode 100644 src/receivers/node_metrics/collector/util.rs create mode 100644 src/receivers/node_metrics/config.rs create mode 100644 src/receivers/node_metrics/convert.rs create mode 100644 src/receivers/node_metrics/mod.rs create mode 100644 src/receivers/node_metrics/receiver.rs create mode 100644 tests/node_metrics_integration_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 930c8fb8..6a736a01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,9 @@ jobs: uses: actions-rs/cargo@v1 with: command: nextest - args: run --target ${{ matrix.target }} --verbose + # node_metrics_receiver pulls in no extra dependencies, so building it here is + # nearly free and keeps its code and tests compiled on every PR. + args: run --target ${{ matrix.target }} --features node_metrics_receiver --verbose - name: run test processor-sdk uses: actions-rs/cargo@v1 with: @@ -83,8 +85,10 @@ jobs: - name: build (no default features) uses: actions-rs/cargo@v1 with: + # node_metrics_receiver is built here too: it reads /proc and /sys through libc, + # whose statfs field widths differ on 32-bit targets. + args: --target ${{ matrix.target }} --no-default-features --features file_receiver,node_metrics_receiver command: build - args: --target ${{ matrix.target }} --no-default-features --features file_receiver # Check formatting with rustfmt formatting: diff --git a/Cargo.toml b/Cargo.toml index ef0f8bcf..7f2cb121 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ default = ["rdkafka", "aws_iam", "file_receiver", "rust_processor"] fluent_receiver = ["dep:rmpv", "dep:rmp-serde"] file_receiver = ["dep:glob", "dep:notify", "dep:notify-debouncer-mini"] kmsg_receiver = ["dep:tempfile"] +node_metrics_receiver = [] pprof = ["dep:pprof"] pyo3 = ["dep:pyo3", "dep:rotel_python_processor_sdk"] rust_processor = ["dep:rotel_rust_processor_sdk", "dep:abi_stable"] diff --git a/README.md b/README.md index 5c22daf0..c03ab221 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,10 @@ Rotel is ideal for resource-constrained environments and applications where mini - Additional exporters: [ClickHouse](#clickhouse-exporter-configuration), [Datadog](#datadog-exporter-configuration), [AWS X-RAY](#aws-x-ray-exporter-configuration), [AWS EMF](#aws-emf-exporter-configuration), and [Kafka](#kafka-exporter-configuration) -- Additional receivers: [File](#file-receiver-configuration), [Fluent](#fluent-receiver-configuration), [Kafka](#kafka-receiver-configuration), and [Ksmsg](#kmsg-receiver-configuration-linux-only) (Linux kernel messages) +- Additional + receivers: [File](#file-receiver-configuration), [Fluent](#fluent-receiver-configuration), [Kafka](#kafka-receiver-configuration), + [Kmsg](#kmsg-receiver-configuration-linux-only) (Linux kernel messages), and + [Node Metrics](#node-metrics-receiver-configuration) (host CPU, memory, disk, network; Linux-oriented) - [Python](#python-processor-sdk) and [Rust](#rust-processor-sdk) processor SDKs Rotel can be easily bundled with popular runtimes as packages. Its Rust implementation ensures minimal resource usage @@ -120,9 +123,13 @@ curl -X POST http://localhost:4318/v1/traces \ - [Parsers](#parsers) - [Offset Persistence](#offset-persistence) - [Example Usage](#example-usage-1) +- [Node Metrics Receiver](#node-metrics-receiver-configuration) + - [Available Metrics](#available-metrics) + - [Example Usage](#example-usage-2) + - [Testing the Node Metrics Receiver](#testing-the-node-metrics-receiver) - [Kmsg Receiver (Linux-only)](#kmsg-receiver-configuration-linux-only) - [Priority Levels](#priority-levels) - - [Example Usage](#example-usage-2) + - [Example Usage](#example-usage-3) - [Testing the Kmsg Receiver](#testing-the-kmsg-receiver) - [Log Record Format](#log-record-format) - [Batch configuration](#batch-configuration) @@ -963,6 +970,480 @@ rotel start \ --otlp-exporter-endpoint "localhost:4317" ``` +### Node Metrics Receiver configuration + +**NOTE**: The Node Metrics Receiver is currently only included when built with the opt-in feature +`--features node_metrics_receiver`. + +The Node Metrics Receiver periodically scrapes system metrics from the host machine and converts them to OpenTelemetry +metrics format. It reads directly from Linux kernel interfaces (`/proc`, `/sys`, syscalls), emitting raw counter values +compatible with Prometheus node_exporter conventions. + +**NOTE**: This receiver targets Linux and requires access to the `/proc` and `/sys` filesystems. Elsewhere only the +`uname`, `time` and `textfile` collectors return data — the collectors that read `/proc` or `/sys` yield nothing — and a +warning is logged at startup. + +Metric names, help strings and the default filesystem-type exclusion list follow Prometheus +[node_exporter](https://github.com/prometheus/node_exporter) (Apache-2.0), so existing dashboards and alerting rules +largely carry over. The collector split differs in one place: `node_boot_time_seconds` is produced by the `time` +collector here, while node_exporter exposes it from its `stat` collector. + +Select the Node Metrics receiver with the option `--receiver node_metrics`. + +| Option | Default | Description | +|--------------------------------------------------|--------------|------------------------------------------------------------------------------------------| +| --node-metrics-receiver-scrape-interval | 60s | Scrape interval as a string time duration (minimum 1 second) | +| --node-metrics-receiver-cpu | true | Enable CPU metrics collection | +| --node-metrics-receiver-loadavg | true | Enable load average metrics collection | +| --node-metrics-receiver-memory | true | Enable memory metrics collection | +| --node-metrics-receiver-network | true | Enable network metrics collection | +| --node-metrics-receiver-filesystem | true | Enable filesystem metrics collection | +| --node-metrics-receiver-uname | true | Enable system info metrics (kernel, hostname) | +| --node-metrics-receiver-stat | true | Enable kernel stat counters (forks, context switches, interrupts, procs running/blocked) | +| --node-metrics-receiver-processes | true | Enable process metrics (kernel limits: threads-max, pid_max) | +| --node-metrics-receiver-diskstats | true | Enable disk I/O statistics (Linux only, /proc/diskstats) | +| --node-metrics-receiver-vmstat | true | Enable virtual memory statistics (/proc/vmstat) | +| --node-metrics-receiver-netstat | true | Enable network statistics (/proc/net/netstat and /proc/net/snmp) | +| --node-metrics-receiver-sockstat | true | Enable socket statistics (/proc/net/sockstat and /proc/net/sockstat6) | +| --node-metrics-receiver-filefd | true | Enable file descriptor statistics (/proc/sys/fs/file-nr) | +| --node-metrics-receiver-cpufreq | true | Enable CPU frequency metrics (/sys/devices/system/cpu/cpu*/cpufreq/) | +| --node-metrics-receiver-thermal-zone | true | Enable thermal zone and cooling device metrics (/sys/class/thermal/) | +| --node-metrics-receiver-nvme | true | Enable NVMe device info metrics (/sys/class/nvme/) | +| --node-metrics-receiver-hwmon | true | Enable hardware monitoring sensor metrics (/sys/class/hwmon/) | +| --node-metrics-receiver-time | true | Enable time metrics (node_time_seconds, node_boot_time_seconds) | +| --node-metrics-receiver-textfile | false | Enable textfile collector for custom Prometheus metrics | +| --node-metrics-receiver-textfile-directory | | Directory of Prometheus-format textfiles (*.prom), or a single .prom file | +| --node-metrics-receiver-filesystem-mount-exclude | | Regex pattern to exclude filesystem mount points, in addition to the built-in excludes | +| --node-metrics-receiver-include-filter | | Regex pattern to include only matching metric names | +| --node-metrics-receiver-exclude-filter | | Regex pattern to exclude matching metric names | +| --node-metrics-receiver-rootfs-path | / | Prefix under which the host root is mounted (for containerized monitoring) | +| --node-metrics-receiver-procfs-path | /proc | Path to procfs mount point (for containerized monitoring) | +| --node-metrics-receiver-sysfs-path | /sys | Path to sysfs mount point (for containerized monitoring) | +| --node-metrics-receiver-service-name | node_metrics | Service name for the OTLP Resource attribute | + +The collector toggles are booleans that take an explicit value, for example `--node-metrics-receiver-cpu false` or +`--node-metrics-receiver-cpu=true`. Passing one of them without a value is an error. The same applies to the +environment variable form, `ROTEL_NODE_METRICS_RECEIVER_CPU=false`. + +`--node-metrics-receiver-rootfs-path` is prefixed onto the procfs and sysfs paths, so `/host` alone yields +`/host/proc` and `/host/sys`. A procfs or sysfs path that is already *under* the rootfs prefix is rejected at startup +with an error naming both flags, rather than silently producing `/host/host/proc` — so pass either the rootfs path or +the already-prefixed individual paths, not both. The comparison is path-component aware, so only a genuine prefix is +rejected: `/host` with `/host/proc` is an error, while `/host` with `/hostile/proc` or `/proc-alt` is accepted and +composes normally into `/host/hostile/proc` and `/host/proc-alt`. The rootfs path applies to the procfs and sysfs paths +only, so `--node-metrics-receiver-textfile-directory` is always used exactly as given. + +A few more startup and runtime behaviours: + +- The scrape interval is a string time duration, so `30s`, `90s` or `2m`, with a 1 second minimum enforced at startup. +- The first scrape runs immediately at startup rather than after one interval. Ticks stay aligned to the original + schedule, so a tick missed while a slow scrape was in progress is skipped rather than delayed or fired back-to-back + as a catch-up burst. +- A scrape that runs longer than the scrape interval, or 30 seconds if the interval is shorter than that, is timed out, + logged, and skipped. (The floor matters: with a 1 second interval, a merely slow host would otherwise time out on + every cycle and the receiver would report nothing at all.) Because a blocking read cannot be aborted, scraping resumes + only once the timed-out scrape actually finishes; each interval until then is skipped and counted as a scrape failure. + An unresponsive filesystem is the usual cause — exclude it with + `--node-metrics-receiver-filesystem-mount-exclude`. +- A collector that panics does not take the receiver down: the panic is logged, counted as a scrape failure, and + scraping continues on the next tick. +- A slow or full pipeline does not block scraping. If the pipeline has not accepted a batch within 10 seconds (or the + scrape interval, whichever is shorter), the batch is abandoned with a warning and counted as refused, and the next + tick scrapes as usual. The warning notes that the batch "may still be delivered", because the hand-off can complete + after the timeout has fired. Without this bound a backed-up exporter would park the receiver indefinitely. +- If no configured exporter accepts metrics, the receiver logs a warning and stays idle without scraping. +- Enabling `--node-metrics-receiver-textfile` without `--node-metrics-receiver-textfile-directory` is a startup error. + Setting the directory without enabling the collector only logs a warning. +- Duplicate entries in the collector list are removed, keeping the first occurrence. This matters only when the config + is built programmatically; the CLI toggles cannot name a collector twice. +- Within one metric, data points that repeat an already seen label set are dropped with a warning and the first + occurrence is kept, since duplicates would break the OTLP single-writer principle. +- Every batch carries the resource attributes `service.name` (from `--node-metrics-receiver-service-name`) and, on + Linux, `os.type`. `host.name` (the local hostname) is added too, and is what keeps otherwise identical series from + different hosts distinct at the backend — it is omitted only if the hostname is not valid UTF-8. +- On shutdown, a batch already in flight is given a short grace window to reach the pipeline; if the window expires, or + the pipeline has already closed, the batch is reported as refused. A scrape still running when cancellation arrives is + discarded without being sent. +- With [internal telemetry](#internal-telemetry) enabled, the receiver reports + `rotel_receiver_accepted_metric_points`, `rotel_receiver_refused_metric_points`, `rotel_receiver_scrape_failures` and + `rotel_receiver_empty_scrapes`, all tagged `receiver="node_metrics"`. Refused metric points are the points of a batch + that could not be handed to the pipeline: the pipeline was closed, the shutdown grace window expired, or the send + timed out. A scrape failure is counted when a scrape exceeds the interval and is timed out, when a collector panics, + and for every cycle skipped while a previous scrape is still running. A scrape that collects nothing at all + increments `rotel_receiver_empty_scrapes` and is logged as a warning; it is not counted as a scrape failure, since an + empty result is a configuration problem rather than a failed scrape. + +#### Available Metrics + +The receiver collects the following metrics (names follow Prometheus node_exporter conventions): + +**CPU** (when `--node-metrics-receiver-cpu=true`): +- `node_cpu_seconds_total` - Seconds the CPUs spent in each mode (with `cpu` and `mode` labels: user, nice, system, idle, iowait, irq, softirq, steal) +- `node_cpu_guest_seconds_total` - Seconds the CPUs spent in guest mode (with `cpu` and `mode` labels: user, nice) + +**NOTE**: Only per-core lines from `/proc/stat` are reported. The aggregate `cpu` line is deliberately skipped, as +Prometheus node_exporter does, so there is no all-core total metric; sum across the `cpu` label instead. + +**Load Average** (when `--node-metrics-receiver-loadavg=true`): +- `node_load1` - 1-minute load average +- `node_load5` - 5-minute load average +- `node_load15` - 15-minute load average + +**Memory** (when `--node-metrics-receiver-memory=true`): +All fields from `/proc/meminfo` are exposed dynamically, including: +- `node_memory_MemTotal_bytes` - Total memory in bytes +- `node_memory_MemFree_bytes` - Free memory in bytes +- `node_memory_MemAvailable_bytes` - Available memory in bytes +- `node_memory_Buffers_bytes` - Buffer memory in bytes +- `node_memory_Cached_bytes` - Cached memory in bytes +- `node_memory_SwapTotal_bytes` - Total swap in bytes +- `node_memory_SwapFree_bytes` - Free swap in bytes +- ... and 30+ additional memory metrics + +**NOTE**: Only fields carrying the `kB` suffix are converted to bytes; those get the `_bytes` name suffix and the unit +`By`. Unitless fields keep their kernel name and carry no unit, for example `HugePages_Total` is exposed as +`node_memory_HugePages_Total`. Parenthesised field names are normalised, so `Active(anon)` becomes +`node_memory_Active_anon_bytes`. + +**Network** (when `--node-metrics-receiver-network=true`, per interface with `device` label): +- `node_network_receive_bytes_total` - Bytes received +- `node_network_receive_packets_total` - Packets received +- `node_network_receive_errs_total` - Receive errors +- `node_network_receive_drop_total` - Receive drops +- `node_network_receive_fifo_total` - Receive FIFO errors +- `node_network_receive_frame_total` - Receive frame errors +- `node_network_receive_compressed_total` - Compressed packets received +- `node_network_receive_multicast_total` - Multicast packets received +- `node_network_transmit_bytes_total` - Bytes transmitted +- `node_network_transmit_packets_total` - Packets transmitted +- `node_network_transmit_errs_total` - Transmit errors +- `node_network_transmit_drop_total` - Transmit drops +- `node_network_transmit_fifo_total` - Transmit FIFO errors +- `node_network_transmit_colls_total` - Transmit collisions +- `node_network_transmit_carrier_total` - Transmit carrier errors +- `node_network_transmit_compressed_total` - Compressed packets transmitted + +**Filesystem** (when `--node-metrics-receiver-filesystem=true`, per mount with `device`, `fstype`, `mountpoint` labels): +- `node_filesystem_size_bytes` - Total filesystem size +- `node_filesystem_free_bytes` - Free space (total free blocks) +- `node_filesystem_avail_bytes` - Available space (free blocks for non-root users) +- `node_filesystem_files` - Total file nodes (inodes) +- `node_filesystem_files_free` - Free file nodes +- `node_filesystem_readonly` - Whether the filesystem is mounted read-only (1 = readonly) + +**NOTE**: +- Virtual filesystems are automatically filtered out. Besides the obvious ones (proc, sysfs, cgroup, overlay), the list + also covers `squashfs`, `iso9660` and `autofs` — so snap and other squashfs-backed mounts, and CD/DVD images, are not + reported. +- Network filesystem types are also excluded by default, because a `statfs` call on an unreachable server can block or + hang. Examples include NFS, SMB/CIFS, Ceph, GlusterFS, Lustre, BeeGFS, GPFS, 9p, AFS, NCP, WebDAV, and remote FUSE + mounts such as sshfs and s3fs; the authoritative list is `collect_filesystem` in + `src/receivers/node_metrics/collector/procfs.rs`. Local FUSE filesystems, such as mergerfs or gocryptfs, are reported + normally. This list of excluded types is not currently overridable: + `--node-metrics-receiver-filesystem-mount-exclude` can only remove further mount points, it cannot bring an excluded + type back. +- Mount points are excluded by prefix, with the match respecting path boundaries, so `/dev` does not exclude + `/developer`. `/proc`, `/sys` and `/dev` are excluded including the mount point itself. `/run/credentials`, + `/run/user`, `/var/lib/docker` and `/var/lib/containers` are excluded only *below* the path: a filesystem mounted + exactly at one of them is a real volume and is reported, which matters most for a dedicated `/var/lib/docker` disk. +- Duplicate mounts (same device + mountpoint) are deduplicated. +- Mount paths containing octal escapes (e.g. `\040` for spaces) are decoded automatically. + +**System/Uname** (when `--node-metrics-receiver-uname=true`): +- `node_uname_info` - System info gauge (labels: sysname, release, version, nodename, machine, domainname [Linux only]) + +**Time** (when `--node-metrics-receiver-time=true`): +- `node_boot_time_seconds` - System boot time in seconds since epoch (from the `btime` line of `/proc/stat`, read at + startup and re-read on every scrape that reads `/proc/stat` — that is, whenever the `cpu`, `stat` or `time` collector + is enabled; the metric is omitted, rather than reported as zero, while the boot time is unknown, and a value that is + not plausible — zero, negative, or in the future — is rejected rather than published) +- `node_time_seconds` - Current system time in seconds since epoch + +**NOTE**: The boot time is not latched, because the kernel's `btime` moves whenever the wall clock is stepped — a device +with no RTC boots near the epoch and then jumps once NTP syncs. `node_boot_time_seconds` therefore tracks the current +kernel value, while the start time stamped on cumulative counters stays latched at the first known value, so a clock +step is not reported downstream as a counter reset. + +**Stat** (when `--node-metrics-receiver-stat=true`): +- `node_forks_total` - Total number of forks since boot +- `node_context_switches_total` - Total number of context switches +- `node_intr_total` - Total number of interrupts serviced (unit `{interrupts}`) +- `node_procs_running` - Number of processes in runnable state +- `node_procs_blocked` - Number of processes blocked waiting for I/O + +**Processes** (when `--node-metrics-receiver-processes=true`): +- `node_processes_max_threads` - Kernel thread limit (`/proc/sys/kernel/threads-max`) +- `node_processes_max_processes` - Kernel PID limit (`/proc/sys/kernel/pid_max`) + +**Disk I/O** (when `--node-metrics-receiver-diskstats=true`, per device with `device` label): +- `node_disk_reads_completed_total` - Total number of reads completed +- `node_disk_reads_merged_total` - Total number of reads merged +- `node_disk_read_bytes_total` - Total bytes read +- `node_disk_read_time_seconds_total` - Total time spent reading +- `node_disk_writes_completed_total` - Total number of writes completed +- `node_disk_writes_merged_total` - Total number of writes merged +- `node_disk_written_bytes_total` - Total bytes written +- `node_disk_write_time_seconds_total` - Total time spent writing +- `node_disk_io_now` - Number of I/Os currently in progress +- `node_disk_io_time_seconds_total` - Total time spent doing I/Os +- `node_disk_io_time_weighted_seconds_total` - Weighted time spent doing I/Os +- `node_disk_discards_completed_total` - Total discards completed (kernel 4.18+) +- `node_disk_discards_merged_total` - Total discards merged (kernel 4.18+) +- `node_disk_discarded_sectors_total` - Total sectors discarded (kernel 4.18+) +- `node_disk_discard_time_seconds_total` - Total time spent discarding (kernel 4.18+) +- `node_disk_flush_requests_total` - Total flush requests completed (kernel 5.5+) +- `node_disk_flush_requests_time_seconds_total` - Total time spent flushing (kernel 5.5+) + +**NOTE**: Virtual devices (ram, zram, loop, fd) and partitions are automatically filtered out. + +**Vmstat** (when `--node-metrics-receiver-vmstat=true`): +All fields from `/proc/vmstat` are exposed dynamically as `node_vmstat_{field}`, including: +- `node_vmstat_pgfault` - Page faults +- `node_vmstat_pgmajfault` - Major page faults +- `node_vmstat_pgpgin` - Pages paged in +- `node_vmstat_pgpgout` - Pages paged out +- `node_vmstat_pswpin` - Pages swapped in +- `node_vmstat_pswpout` - Pages swapped out +- ... and 100+ additional vmstat metrics + +**NOTE**: The vmstat collector emits all fields from `/proc/vmstat`, which can be 100+ metrics on modern kernels. +To reduce cardinality, either disable the collector entirely (`--node-metrics-receiver-vmstat false`) +or use an include filter that lists all desired metric prefixes across all collectors, e.g.: +`--node-metrics-receiver-include-filter "^node_(vmstat_(oom_kill|pgpg|pswp|pg.*fault)|load|memory_|cpu_)"` + +**Netstat** (when `--node-metrics-receiver-netstat=true`): +TCP/IP statistics from `/proc/net/netstat` and `/proc/net/snmp` as `node_netstat_{Protocol}_{Field}`: +- `node_netstat_TcpExt_TCPTimeouts` - TCP timeouts +- `node_netstat_TcpExt_TCPRetransFail` - TCP retransmit failures +- `node_netstat_TcpExt_SyncookiesSent` - SYN cookies sent +- `node_netstat_IpExt_InOctets` - IP octets received +- `node_netstat_IpExt_OutOctets` - IP octets sent +- `node_netstat_Tcp_CurrEstab` - TCP connections currently established (from /proc/net/snmp) +- ... and many more TCP/IP statistics + +**NOTE**: Like vmstat, the netstat collector emits every field of both files, which is 200+ metrics — considerably more +than node_exporter, which whitelists a subset by default. On a backend billed per series, either disable the collector +or narrow it with an include filter such as +`--node-metrics-receiver-include-filter "^node_netstat_(Tcp|Udp)_"`. + +**Sockstat** (when `--node-metrics-receiver-sockstat=true`): +Socket allocation statistics from `/proc/net/sockstat` and `/proc/net/sockstat6`: +- `node_sockstat_sockets_used` - Total sockets in use +- `node_sockstat_TCP_inuse` - TCP sockets in use +- `node_sockstat_TCP_orphan` - TCP orphaned sockets +- `node_sockstat_TCP_tw` - TCP TIME_WAIT sockets +- `node_sockstat_TCP_alloc` - TCP sockets allocated +- `node_sockstat_TCP_mem` - TCP memory pages +- `node_sockstat_TCP_mem_bytes` - TCP memory in bytes +- `node_sockstat_UDP_inuse` - UDP sockets in use +- `node_sockstat_UDP_mem` - UDP memory pages +- `node_sockstat_UDP_mem_bytes` - UDP memory in bytes +- `node_sockstat_RAW_inuse` - RAW sockets in use +- `node_sockstat_TCP6_inuse` - TCP6 sockets in use (from sockstat6) +- ... and additional IPv6 socket statistics + +**Filefd** (when `--node-metrics-receiver-filefd=true`): +- `node_filefd_allocated` - Number of allocated file descriptors (unit `{file_descriptors}`) +- `node_filefd_maximum` - Maximum number of file descriptors allowed (unit `{file_descriptors}`) + +**CPU Frequency** (when `--node-metrics-receiver-cpufreq=true`, read from +`/sys/devices/system/cpu/cpu*/cpufreq/`, per CPU with `cpu` label): +- `node_cpu_scaling_frequency_hertz` - Current CPU scaling frequency +- `node_cpu_scaling_frequency_min_hertz` - Minimum CPU scaling frequency +- `node_cpu_scaling_frequency_max_hertz` - Maximum CPU scaling frequency +- `node_cpu_frequency_hertz` - Current CPU frequency (from cpuinfo) +- `node_cpu_frequency_min_hertz` - Minimum CPU frequency (from cpuinfo) +- `node_cpu_frequency_max_hertz` - Maximum CPU frequency (from cpuinfo) + +**NOTE**: Only directories named `cpu` followed by digits (`cpu0`, `cpu1`, ...) are scanned, so entries such as +`cpuidle` and `cpufreq` under the same parent are ignored. + +**Thermal Zone** (when `--node-metrics-receiver-thermal-zone=true`): +- `node_thermal_zone_temp` - Thermal zone temperature in Celsius (with `zone` and `type` labels) +- `node_cooling_device_cur_state` - Current cooling device state (with `name` and `type` labels) +- `node_cooling_device_max_state` - Maximum cooling device state (with `name` and `type` labels) + +**NVMe** (when `--node-metrics-receiver-nvme=true`): +- `node_nvme_info` - NVMe device info gauge (with `device`, `firmware_revision`, `model`, `serial`, `state` labels) + +**NOTE**: Only entries named `nvme` followed by digits (`nvme0`, `nvme1`, ...) are scanned, so any other entry under +`/sys/class/nvme` is ignored. + +**Hwmon** (when `--node-metrics-receiver-hwmon=true`, sensor readings carry `chip` and `sensor` labels): +- `node_hwmon_temp_celsius` - Temperature sensor reading +- `node_hwmon_temp_max_celsius` - Temperature max threshold +- `node_hwmon_temp_crit_celsius` - Temperature critical threshold +- `node_hwmon_in_volts` - Voltage sensor reading +- `node_hwmon_fan_rpm` - Fan speed, in revolutions per minute (unit `{rev}/min`) +- `node_hwmon_power_watts` - Power consumption in watts +- `node_hwmon_curr_amps` - Current in amps (converted from milliamps) +- `node_hwmon_chip_names` - Annotation gauge carrying the human-readable chip model (with `chip` and `chip_name` labels) +- `node_hwmon_sensor_label` - Annotation gauge carrying the human-readable sensor label, when the chip provides one (with `chip`, `sensor` and `label` labels) + +**NOTE**: The `sensor` label is the raw sysfs name (`temp1`, `in0`), matching Prometheus node_exporter — it is unique +within a chip, so two sensors can never collapse onto one series, and it does not move when a driver gains or changes a +label file. Where the chip provides human-readable text, that is published separately by `node_hwmon_sensor_label` and +can be joined on `chip` and `sensor`. + +**NOTE**: The `chip` label is derived from the backing device of the hwmon instance, so two chips reporting the same +model name (for example two NVMe drives, both named `nvme`) do not collapse onto the same label set. The model name +itself is published separately by `node_hwmon_chip_names`, which can be joined on `chip`. When an hwmon instance has no +backing `device`, the identity falls back to the chip model name combined with the `hwmonN` directory, which is still +unique within a host but is not stable across reboots, since the hwmon index depends on module load order. + +**NOTE**: The `sensor` label uses the human-readable name from the sysfs `*_label` file when available +(e.g., "Core 0", "Package id 0"), falling back to the raw sensor name (e.g., "temp1"). + +**NOTE**: Only the temperature, voltage, fan, power and current sensor kinds are read. Any other kind exposed by a chip +is skipped with a debug log. + +**Textfile** (when `--node-metrics-receiver-textfile=true`): +Custom metrics from Prometheus-format text files. `--node-metrics-receiver-textfile-directory` accepts either a +directory, in which case every `*.prom` file in it is read in sorted filename order, or the path of a single `.prom` +file. The directory scan is not recursive, so subdirectories are never descended into, and an entry that is itself a +directory named `something.prom` is skipped silently. Symlinks are followed, so a symlinked `.prom` file is collected +like any other. +The `--node-metrics-receiver-include-filter` and `--node-metrics-receiver-exclude-filter` patterns apply to textfile +metrics just as they do to the built-in ones. Accepted input: +- Metric lines: `metric_name{label="value"} 42.5` +- TYPE comments: `# TYPE metric_name counter|gauge` (used to set the metric type) +- HELP comments: `# HELP metric_name Description` (propagated as the OTLP metric description) + +Support for the exposition format is deliberately partial: +- Only the `counter` and `gauge` types are supported. `histogram`, `summary` and `untyped` are treated as gauges, as is + any metric with no `# TYPE` line. +- Trailing per-sample timestamps are accepted but ignored; every data point is stamped with the scrape time. +- `# TYPE` and `# HELP` are scoped to the file they appear in, so the same metric name can be declared differently in + two files. Conflicting `# TYPE` declarations produce two separate OTLP metrics sharing that name, one gauge and one + sum, because metrics are grouped by name *and* type. Within each of those groups the unit and description of the first + sample seen win. +- A textfile larger than 10 MiB is skipped, guarding against a runaway writer script, and its + `node_textfile_scrape_error` is set to 1. The limit is enforced while reading, not from the file's reported size, so a + file that grows between the two cannot slip past it. Its `node_textfile_mtime_seconds` is still emitted even though the + file itself is skipped, so a writer that has stalled stays observable. +- At most 100,000 samples are taken from the textfile directory per scrape. Beyond that the remainder of the file is + dropped, a warning names how many samples were kept, and no further files are read — a bound on memory that the + per-file size limit alone does not give, since a small file can expand into a great many samples. +- Files that are not regular files are skipped silently: a FIFO named `*.prom` would otherwise block the scrape + indefinitely when opened. A symlink to a regular file is followed and collected; a `.prom` entry that cannot be read + at all, such as a dangling symlink, sets `node_textfile_scrape_error` to 1. +- A line with a malformed label — an invalid or empty label name, or a value that is not `key="value"` — is rejected in + full and counted as a malformed line, rather than being emitted without that label. Dropping a label would change the + series identity and silently merge samples that are meant to be distinct. A label name repeated within one line keeps + its first value. + +Additionally emits: +- `node_textfile_scrape_error` - Per-file error indicator (0 = success, 1 = the file could not be read, exceeded the + 10 MiB size limit, or contained a malformed line; with `filename` label). A textfile directory that is missing or + cannot be read yields a single error point whose `filename` label holds the configured path rather than a basename. +- `node_textfile_mtime_seconds` - Modification time of the file in seconds since epoch (with `filename` label), useful for spotting a writer script that has stopped updating + +Example textfile (`/var/lib/node_exporter/textfile/custom.prom`): +``` +# HELP my_app_version Application version info +# TYPE my_app_version gauge +my_app_version{version="1.2.3"} 1 +# TYPE my_app_requests_total counter +my_app_requests_total{endpoint="/api"} 12345 +``` + +#### Example Usage + +Basic usage collecting all metrics and exporting to OTLP: + +```shell +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-scrape-interval 30s \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +Collecting only CPU and memory metrics: + +```shell +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-loadavg false \ + --node-metrics-receiver-network false \ + --node-metrics-receiver-filesystem false \ + --node-metrics-receiver-uname false \ + --node-metrics-receiver-stat false \ + --node-metrics-receiver-processes false \ + --node-metrics-receiver-diskstats false \ + --node-metrics-receiver-vmstat false \ + --node-metrics-receiver-netstat false \ + --node-metrics-receiver-sockstat false \ + --node-metrics-receiver-filefd false \ + --node-metrics-receiver-cpufreq false \ + --node-metrics-receiver-thermal-zone false \ + --node-metrics-receiver-nvme false \ + --node-metrics-receiver-hwmon false \ + --node-metrics-receiver-time false \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +Using filters to only collect memory-related metrics: + +```shell +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-include-filter "^node_memory_" \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +Monitoring the host system from inside a container. Mount the host root, and point the receiver at it with a single +rootfs path: + +```shell +# docker run -v /:/host:ro,rslave ... +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-rootfs-path /host \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +**NOTE**: mounting only `/proc` and `/sys` is enough for every collector *except* `filesystem`. Filesystem metrics need +the host root as well, because the mount table names host paths that `statfs` has to be able to resolve — the rootfs +path is what makes that work. Without it the receiver would report the container's own mounts labelled as though they +were the host's, so prefer the form above over setting `--node-metrics-receiver-procfs-path` and +`--node-metrics-receiver-sysfs-path` individually. The mount table is read from PID 1 (`/proc/1/mounts`) rather than +`/proc/self/mounts` for the same reason: the latter is always the reading process's own mount namespace, whatever +procfs it is read from. + +**NOTE**: The same reasoning applies to a hardened systemd unit on the host. With `PrivateMounts=`, `PrivateTmp=` or +`ProtectSystem=`, rotel's mount namespace differs from PID 1's: the mount table names the host's paths while `statfs` +resolves inside the unit's namespace, so a metric labelled `mountpoint="/tmp"` would describe the unit's private tmpfs. +Prometheus `node_exporter` behaves the same way. Run the receiver without mount-namespace isolation if filesystem +metrics matter. + +Using the textfile collector for custom Prometheus metrics: + +```shell +rotel start \ + --receiver node_metrics \ + --node-metrics-receiver-textfile true \ + --node-metrics-receiver-textfile-directory /var/lib/node_exporter/textfile \ + --exporter otlp \ + --otlp-exporter-endpoint "localhost:4317" +``` + +#### Testing the Node Metrics Receiver + +To run integration tests that read the real `/proc` and `/sys` of a Linux host: + +```shell +NODE_METRICS_INTEGRATION_TESTS=true cargo test --test node_metrics_integration_tests --features node_metrics_receiver +``` + +The tests are skipped unless `NODE_METRICS_INTEGRATION_TESTS=true` is set and the `node_metrics_receiver` feature is +enabled. Collectors for optional hardware (thermal zones, hwmon sensors, NVMe devices, cpufreq, diskstats) are absent in +many containers and virtual machines, so the tests for those accept an empty result and require only that any value +produced is finite — asserting a particular piece of hardware is present would make them flaky by design. + ### Kmsg Receiver configuration (Linux-only) **NOTE**: The Kmsg Receiver is Linux-only and must be enabled with the feature flag `--features kmsg_receiver`. @@ -1153,10 +1634,10 @@ to receive data via OTLP and consume from Kafka topics at the same time. The following configuration parameters enable multiple receivers: -| Option | Default | Options | -|-------------|---------|---------------------------------------------| -| --receiver | otlp | otlp, kafka, fluent, file, kmsg | -| --receivers | | comma-separated list (otlp,kafka,file,kmsg) | +| Option | Default | Options | +|-------------|---------|----------------------------------------------------------| +| --receiver | otlp | otlp, kafka, fluent, file, kmsg, node_metrics | +| --receivers | | comma-separated list (otlp,kafka,file,kmsg,node_metrics) | **Important Notes:** diff --git a/build.rs b/build.rs index 1dc44f0f..eaae52f7 100644 --- a/build.rs +++ b/build.rs @@ -3,9 +3,11 @@ use std::io::Result; fn main() -> Result<()> { println!("cargo::rustc-check-cfg=cfg(kafka_integration_tests, values(\"true\"))"); println!("cargo::rustc-check-cfg=cfg(kmsg_integration_tests, values(\"true\"))"); + println!("cargo::rustc-check-cfg=cfg(node_metrics_integration_tests, values(\"true\"))"); println!("cargo::rustc-check-cfg=cfg(rust_processor_integration_tests, values(\"true\"))"); println!("cargo::rerun-if-env-changed=KAFKA_INTEGRATION_TESTS"); println!("cargo::rerun-if-env-changed=KMSG_INTEGRATION_TESTS"); + println!("cargo::rerun-if-env-changed=NODE_METRICS_INTEGRATION_TESTS"); println!("cargo::rerun-if-env-changed=RUST_PROCESSOR_INTEGRATION_TESTS"); println!("cargo::rerun-if-changed=proto/datadog"); prost_build::compile_protos(&["proto/datadog/agent_payload.proto"], &["proto/datadog"])?; @@ -16,6 +18,9 @@ fn main() -> Result<()> { let env = std::env::var("KMSG_INTEGRATION_TESTS").unwrap_or("false".to_string()); println!("cargo:rustc-cfg=kmsg_integration_tests=\"{}\"", env); + let env = std::env::var("NODE_METRICS_INTEGRATION_TESTS").unwrap_or("false".to_string()); + println!("cargo:rustc-cfg=node_metrics_integration_tests=\"{}\"", env); + let env = std::env::var("RUST_PROCESSOR_INTEGRATION_TESTS").unwrap_or("false".to_string()); println!( "cargo:rustc-cfg=rust_processor_integration_tests=\"{}\"", diff --git a/src/init/activation.rs b/src/init/activation.rs index a2797ef7..d4967e7b 100644 --- a/src/init/activation.rs +++ b/src/init/activation.rs @@ -72,6 +72,8 @@ fn all_traces_receivers_disabled(rc: &HashMap) -> bool ReceiverConfig::File(_) => {} // File receiver doesn't handle traces #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] ReceiverConfig::Kmsg(_) => {} // Kmsg receiver doesn't handle traces + #[cfg(feature = "node_metrics_receiver")] + ReceiverConfig::NodeMetrics(_) => {} // Node metrics receiver doesn't handle traces } } true @@ -98,6 +100,8 @@ fn all_metrics_receivers_disabled(rc: &HashMap) -> boo ReceiverConfig::File(_) => {} // File receiver doesn't handle metrics #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] ReceiverConfig::Kmsg(_) => {} // Kmsg receiver doesn't handle metrics + #[cfg(feature = "node_metrics_receiver")] + ReceiverConfig::NodeMetrics(_) => return false, // Node metrics receiver handles metrics } } true @@ -123,7 +127,78 @@ fn all_logs_receivers_disabled(rc: &HashMap) -> bool { ReceiverConfig::File(_) => return false, // File receiver handles logs #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] ReceiverConfig::Kmsg(_) => return false, // Kmsg receiver handles logs + #[cfg(feature = "node_metrics_receiver")] + ReceiverConfig::NodeMetrics(_) => {} // Node metrics receiver doesn't handle logs } } true } + +#[cfg(all(test, feature = "node_metrics_receiver"))] +mod tests { + use super::*; + use crate::init::config::ExporterConfig; + use crate::receivers::node_metrics::config::NodeMetricsConfig; + + /// A receiver map holding only the node metrics receiver + fn node_metrics_only() -> HashMap { + HashMap::from([( + Receiver::NodeMetrics, + ReceiverConfig::NodeMetrics(NodeMetricsConfig::default()), + )]) + } + + /// The node metrics receiver is the only source of metrics in its own pipeline. + /// + /// This single match arm decides whether the metrics pipeline is activated at all: + /// folded in with the neighbouring receivers that do not produce metrics, the + /// receiver would scrape forever into a pipeline that was never started. + #[test] + fn test_node_metrics_receiver_keeps_the_metrics_pipeline_enabled() { + let receivers = node_metrics_only(); + + assert!( + !all_metrics_receivers_disabled(&receivers), + "a node metrics receiver must keep the metrics pipeline enabled" + ); + + // The `false` above has to come from the node metrics arm, not from an + // unconditionally false result: with no receivers at all, metrics stay disabled. + assert!(all_metrics_receivers_disabled(&HashMap::new())); + + // It produces neither traces nor logs, so those pipelines stay disabled — the + // arms that would activate a whole pipeline for nothing. + assert!(all_traces_receivers_disabled(&receivers)); + assert!(all_logs_receivers_disabled(&receivers)); + } + + /// The same decision as seen by the agent, which acts on `TelemetryActivation`. + #[test] + fn test_activation_with_only_a_node_metrics_receiver() { + let exporters = ExporterConfigs { + metrics: vec![ExporterConfig::Blackhole], + ..Default::default() + }; + + let activation = TelemetryActivation::from_config(&node_metrics_only(), &exporters, false); + + // Metrics are the only pipeline this receiver feeds: it has both a receiver and + // an exporter, so it is the only one that comes up. + assert_eq!(activation.metrics, TelemetryState::Active); + assert_eq!(activation.traces, TelemetryState::Disabled); + assert_eq!(activation.logs, TelemetryState::Disabled); + } + + /// Without a metrics exporter there is nowhere to send scrapes, which is reported as + /// `NoListeners` rather than as a disabled receiver. + #[test] + fn test_activation_without_a_metrics_exporter() { + let activation = TelemetryActivation::from_config( + &node_metrics_only(), + &ExporterConfigs::default(), + false, + ); + + assert_eq!(activation.metrics, TelemetryState::NoListeners); + } +} diff --git a/src/init/agent.rs b/src/init/agent.rs index 7a575400..26f99031 100644 --- a/src/init/agent.rs +++ b/src/init/agent.rs @@ -31,6 +31,8 @@ use crate::receivers::kafka::offset_ack_committer::KafkaOffsetCommitter; use crate::receivers::kafka::receiver::KafkaReceiver; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] use crate::receivers::kmsg::receiver::KmsgReceiver; +#[cfg(feature = "node_metrics_receiver")] +use crate::receivers::node_metrics::NodeMetricsReceiver; use crate::receivers::otlp::otlp_grpc::OTLPGrpcServer; use crate::receivers::otlp::otlp_http::OTLPHttpServer; use crate::receivers::otlp_output::OTLPOutput; @@ -1082,6 +1084,17 @@ impl Agent { } }); } + #[cfg(feature = "node_metrics_receiver")] + ReceiverConfig::NodeMetrics(config) => { + let node_metrics_receiver = + NodeMetricsReceiver::new(config.clone(), metrics_output.clone())?; + + // Spawned directly into the shared task set rather than into its own + // supervised set: this receiver owns a single task, so there is + // nothing to supervise or drain. It runs until cancelled, so an early + // completion here legitimately means something went wrong. + node_metrics_receiver.start(&mut receivers_task_set, &receivers_cancel)?; + } } } diff --git a/src/init/args.rs b/src/init/args.rs index 6a00d8c1..e182cd41 100644 --- a/src/init/args.rs +++ b/src/init/args.rs @@ -13,6 +13,8 @@ use crate::init::kafka_exporter::KafkaExporterArgs; use crate::init::kafka_receiver::KafkaReceiverArgs; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] use crate::init::kmsg_receiver::KmsgReceiverArgs; +#[cfg(feature = "node_metrics_receiver")] +use crate::init::node_metrics_receiver::NodeMetricsReceiverArgs; use crate::init::otlp_exporter::OTLPExporterArgs; use crate::init::otlp_receiver::OTLPReceiverArgs; #[cfg(feature = "prometheus")] @@ -82,6 +84,10 @@ pub struct AgentRun { #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] pub kmsg_receiver: KmsgReceiverArgs, + #[command(flatten)] + #[cfg(feature = "node_metrics_receiver")] + pub node_metrics_receiver: NodeMetricsReceiverArgs, + /// Single receiver (type) #[arg(value_enum, long, env = "ROTEL_RECEIVER")] pub receiver: Option, @@ -214,6 +220,8 @@ impl Default for AgentRun { file_receiver: FileReceiverArgs::default(), #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] kmsg_receiver: KmsgReceiverArgs::default(), + #[cfg(feature = "node_metrics_receiver")] + node_metrics_receiver: NodeMetricsReceiverArgs::default(), otlp_with_trace_processor: Vec::new(), otlp_with_logs_processor: Vec::new(), otlp_with_metrics_processor: Vec::new(), @@ -317,6 +325,9 @@ pub enum Receiver { File, #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] Kmsg, + #[cfg(feature = "node_metrics_receiver")] + #[clap(name = "node_metrics")] + NodeMetrics, } impl FromStr for Receiver { @@ -332,6 +343,8 @@ impl FromStr for Receiver { "file" => Ok(Receiver::File), #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] "kmsg" => Ok(Receiver::Kmsg), + #[cfg(feature = "node_metrics_receiver")] + "node_metrics" => Ok(Receiver::NodeMetrics), _ => Err("Unknown receiver"), } } diff --git a/src/init/config.rs b/src/init/config.rs index 11fff59e..9d0fa30d 100644 --- a/src/init/config.rs +++ b/src/init/config.rs @@ -28,6 +28,8 @@ use crate::receivers::fluent::config::FluentReceiverConfig; use crate::receivers::kafka::config::KafkaReceiverConfig; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] use crate::receivers::kmsg::config::KmsgReceiverConfig; +#[cfg(feature = "node_metrics_receiver")] +use crate::receivers::node_metrics::config::NodeMetricsConfig; use crate::receivers::otlp::OTLPReceiverConfig; use figment::{Figment, providers::Env}; use gethostname::gethostname; @@ -214,6 +216,8 @@ pub(crate) enum ReceiverConfig { File(FileReceiverConfig), #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] Kmsg(KmsgReceiverConfig), + #[cfg(feature = "node_metrics_receiver")] + NodeMetrics(NodeMetricsConfig), } impl TryIntoConfig for ExporterArgs { @@ -650,6 +654,10 @@ fn get_receiver_config(config: &AgentRun, receiver: Receiver) -> Result ReceiverConfig::File(config.file_receiver.build_config()), #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] Receiver::Kmsg => ReceiverConfig::Kmsg(config.kmsg_receiver.build_config()), + #[cfg(feature = "node_metrics_receiver")] + Receiver::NodeMetrics => { + ReceiverConfig::NodeMetrics(config.node_metrics_receiver.build_config()?) + } }) } diff --git a/src/init/mod.rs b/src/init/mod.rs index 646d4534..fd1935ac 100644 --- a/src/init/mod.rs +++ b/src/init/mod.rs @@ -17,6 +17,9 @@ mod file_receiver; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] mod kmsg_receiver; +#[cfg(feature = "node_metrics_receiver")] +mod node_metrics_receiver; + mod awsemf_exporter; mod clickhouse_exporter; mod datadog_exporter; diff --git a/src/init/node_metrics_receiver.rs b/src/init/node_metrics_receiver.rs new file mode 100644 index 00000000..91b7342b --- /dev/null +++ b/src/init/node_metrics_receiver.rs @@ -0,0 +1,875 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! CLI integration for the node metrics receiver + +use crate::receivers::node_metrics::collector::matches_path_prefix; +use crate::receivers::node_metrics::config::{ + Collector, DEFAULT_PROCFS_PATH, DEFAULT_SYSFS_PATH, NodeMetricsConfig, +}; +use clap::Args; +use serde::Deserialize; +use std::time::Duration; +use tower::BoxError; + +#[derive(Debug, Args, Clone, Deserialize)] +#[serde(default)] +pub struct NodeMetricsReceiverArgs { + /// Interval between scrapes, as a time duration (minimum 1s) + #[arg( + id("NODE_METRICS_RECEIVER_SCRAPE_INTERVAL"), + long("node-metrics-receiver-scrape-interval"), + env = "ROTEL_NODE_METRICS_RECEIVER_SCRAPE_INTERVAL", + default_value = "60s", + value_parser = humantime::parse_duration + )] + #[serde(with = "humantime_serde")] + pub scrape_interval: Duration, + + /// Enable CPU metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_CPU"), + long("node-metrics-receiver-cpu"), + env = "ROTEL_NODE_METRICS_RECEIVER_CPU", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub cpu: bool, + + /// Enable load average metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_LOADAVG"), + long("node-metrics-receiver-loadavg"), + env = "ROTEL_NODE_METRICS_RECEIVER_LOADAVG", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub loadavg: bool, + + /// Enable memory metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_MEMORY"), + long("node-metrics-receiver-memory"), + env = "ROTEL_NODE_METRICS_RECEIVER_MEMORY", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub memory: bool, + + /// Enable network metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_NETWORK"), + long("node-metrics-receiver-network"), + env = "ROTEL_NODE_METRICS_RECEIVER_NETWORK", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub network: bool, + + /// Enable filesystem metrics collection + #[arg( + id("NODE_METRICS_RECEIVER_FILESYSTEM"), + long("node-metrics-receiver-filesystem"), + env = "ROTEL_NODE_METRICS_RECEIVER_FILESYSTEM", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub filesystem: bool, + + /// Enable system/uname info metrics collection (kernel info, hostname) + #[arg( + id("NODE_METRICS_RECEIVER_UNAME"), + long("node-metrics-receiver-uname"), + env = "ROTEL_NODE_METRICS_RECEIVER_UNAME", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub uname: bool, + + /// Enable kernel stat counters (forks, context switches, interrupts, procs running/blocked from /proc/stat) + #[arg( + id("NODE_METRICS_RECEIVER_STAT"), + long("node-metrics-receiver-stat"), + env = "ROTEL_NODE_METRICS_RECEIVER_STAT", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub stat: bool, + + /// Enable process metrics collection (kernel limits: threads-max, pid_max) + #[arg( + id("NODE_METRICS_RECEIVER_PROCESSES"), + long("node-metrics-receiver-processes"), + env = "ROTEL_NODE_METRICS_RECEIVER_PROCESSES", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub processes: bool, + + /// Enable disk I/O statistics collection (Linux only, reads /proc/diskstats) + #[arg( + id("NODE_METRICS_RECEIVER_DISKSTATS"), + long("node-metrics-receiver-diskstats"), + env = "ROTEL_NODE_METRICS_RECEIVER_DISKSTATS", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub diskstats: bool, + + /// Enable virtual memory statistics collection (/proc/vmstat) + #[arg( + id("NODE_METRICS_RECEIVER_VMSTAT"), + long("node-metrics-receiver-vmstat"), + env = "ROTEL_NODE_METRICS_RECEIVER_VMSTAT", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub vmstat: bool, + + /// Enable network statistics collection (/proc/net/netstat and /proc/net/snmp) + #[arg( + id("NODE_METRICS_RECEIVER_NETSTAT"), + long("node-metrics-receiver-netstat"), + env = "ROTEL_NODE_METRICS_RECEIVER_NETSTAT", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub netstat: bool, + + /// Enable socket statistics collection (/proc/net/sockstat and /proc/net/sockstat6) + #[arg( + id("NODE_METRICS_RECEIVER_SOCKSTAT"), + long("node-metrics-receiver-sockstat"), + env = "ROTEL_NODE_METRICS_RECEIVER_SOCKSTAT", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub sockstat: bool, + + /// Enable file descriptor statistics collection (/proc/sys/fs/file-nr) + #[arg( + id("NODE_METRICS_RECEIVER_FILEFD"), + long("node-metrics-receiver-filefd"), + env = "ROTEL_NODE_METRICS_RECEIVER_FILEFD", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub filefd: bool, + + /// Enable CPU frequency metrics collection (/sys/devices/system/cpu/cpu*/cpufreq/) + #[arg( + id("NODE_METRICS_RECEIVER_CPUFREQ"), + long("node-metrics-receiver-cpufreq"), + env = "ROTEL_NODE_METRICS_RECEIVER_CPUFREQ", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub cpufreq: bool, + + /// Enable thermal zone and cooling device metrics (/sys/class/thermal/) + #[arg( + id("NODE_METRICS_RECEIVER_THERMAL_ZONE"), + long("node-metrics-receiver-thermal-zone"), + env = "ROTEL_NODE_METRICS_RECEIVER_THERMAL_ZONE", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub thermal_zone: bool, + + /// Enable NVMe device info metrics (/sys/class/nvme/) + #[arg( + id("NODE_METRICS_RECEIVER_NVME"), + long("node-metrics-receiver-nvme"), + env = "ROTEL_NODE_METRICS_RECEIVER_NVME", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub nvme: bool, + + /// Enable hardware monitoring sensor metrics (/sys/class/hwmon/) + #[arg( + id("NODE_METRICS_RECEIVER_HWMON"), + long("node-metrics-receiver-hwmon"), + env = "ROTEL_NODE_METRICS_RECEIVER_HWMON", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub hwmon: bool, + + /// Enable time metrics (node_time_seconds, node_boot_time_seconds) + #[arg( + id("NODE_METRICS_RECEIVER_TIME"), + long("node-metrics-receiver-time"), + env = "ROTEL_NODE_METRICS_RECEIVER_TIME", + action = clap::ArgAction::Set, + default_value = "true" + )] + pub time: bool, + + /// Enable textfile collector for custom Prometheus-format metrics + #[arg( + id("NODE_METRICS_RECEIVER_TEXTFILE"), + long("node-metrics-receiver-textfile"), + env = "ROTEL_NODE_METRICS_RECEIVER_TEXTFILE", + action = clap::ArgAction::Set, + default_value = "false" + )] + pub textfile: bool, + + /// Directory or file path for Prometheus-format textfiles (*.prom) + #[arg( + id("NODE_METRICS_RECEIVER_TEXTFILE_DIRECTORY"), + long("node-metrics-receiver-textfile-directory"), + env = "ROTEL_NODE_METRICS_RECEIVER_TEXTFILE_DIRECTORY" + )] + pub textfile_directory: Option, + + /// Root filesystem path prefix, auto-composed onto procfs/sysfs paths + /// (e.g. --node-metrics-receiver-rootfs-path=/host sets procfs to /host/proc and sysfs to /host/sys) + #[arg( + id("NODE_METRICS_RECEIVER_ROOTFS_PATH"), + long("node-metrics-receiver-rootfs-path"), + env = "ROTEL_NODE_METRICS_RECEIVER_ROOTFS_PATH", + default_value = "/" + )] + pub rootfs_path: String, + + /// Path to procfs mount point (for containerized monitoring) + #[arg( + id("NODE_METRICS_RECEIVER_PROCFS_PATH"), + long("node-metrics-receiver-procfs-path"), + env = "ROTEL_NODE_METRICS_RECEIVER_PROCFS_PATH", + default_value = DEFAULT_PROCFS_PATH + )] + pub procfs_path: String, + + /// Path to sysfs mount point (for containerized monitoring) + #[arg( + id("NODE_METRICS_RECEIVER_SYSFS_PATH"), + long("node-metrics-receiver-sysfs-path"), + env = "ROTEL_NODE_METRICS_RECEIVER_SYSFS_PATH", + default_value = DEFAULT_SYSFS_PATH + )] + pub sysfs_path: String, + + /// Service name for the OTLP Resource attribute + #[arg( + id("NODE_METRICS_RECEIVER_SERVICE_NAME"), + long("node-metrics-receiver-service-name"), + env = "ROTEL_NODE_METRICS_RECEIVER_SERVICE_NAME", + default_value = "node_metrics" + )] + pub service_name: String, + + /// Regex pattern to exclude filesystem mount points, in addition to the built-in exclusions + #[arg( + id("NODE_METRICS_RECEIVER_FILESYSTEM_MOUNT_EXCLUDE"), + long("node-metrics-receiver-filesystem-mount-exclude"), + env = "ROTEL_NODE_METRICS_RECEIVER_FILESYSTEM_MOUNT_EXCLUDE" + )] + pub filesystem_mount_exclude: Option, + + /// Regex pattern to include only matching metric names + #[arg( + id("NODE_METRICS_RECEIVER_INCLUDE_FILTER"), + long("node-metrics-receiver-include-filter"), + env = "ROTEL_NODE_METRICS_RECEIVER_INCLUDE_FILTER" + )] + pub include_filter: Option, + + /// Regex pattern to exclude matching metric names + #[arg( + id("NODE_METRICS_RECEIVER_EXCLUDE_FILTER"), + long("node-metrics-receiver-exclude-filter"), + env = "ROTEL_NODE_METRICS_RECEIVER_EXCLUDE_FILTER" + )] + pub exclude_filter: Option, +} + +impl Default for NodeMetricsReceiverArgs { + fn default() -> Self { + Self { + scrape_interval: Duration::from_secs(60), + cpu: true, + loadavg: true, + memory: true, + network: true, + filesystem: true, + uname: true, + stat: true, + processes: true, + diskstats: true, + vmstat: true, + netstat: true, + sockstat: true, + filefd: true, + cpufreq: true, + thermal_zone: true, + nvme: true, + hwmon: true, + time: true, + textfile: false, + textfile_directory: None, + rootfs_path: "/".to_string(), + procfs_path: DEFAULT_PROCFS_PATH.to_string(), + sysfs_path: DEFAULT_SYSFS_PATH.to_string(), + service_name: "node_metrics".to_string(), + filesystem_mount_exclude: None, + include_filter: None, + exclude_filter: None, + } + } +} + +impl NodeMetricsReceiverArgs { + /// Build a NodeMetricsConfig from the CLI args + pub fn build_config(&self) -> Result { + // Collect enabled collectors + let mut collectors = Vec::new(); + if self.cpu { + collectors.push(Collector::Cpu); + } + if self.loadavg { + collectors.push(Collector::Loadavg); + } + if self.memory { + collectors.push(Collector::Memory); + } + if self.network { + collectors.push(Collector::Network); + } + if self.filesystem { + collectors.push(Collector::Filesystem); + } + if self.uname { + collectors.push(Collector::Uname); + } + if self.stat { + collectors.push(Collector::Stat); + } + if self.processes { + collectors.push(Collector::Processes); + } + if self.diskstats { + collectors.push(Collector::Diskstats); + } + if self.vmstat { + collectors.push(Collector::Vmstat); + } + if self.netstat { + collectors.push(Collector::Netstat); + } + if self.sockstat { + collectors.push(Collector::Sockstat); + } + if self.filefd { + collectors.push(Collector::Filefd); + } + if self.cpufreq { + collectors.push(Collector::Cpufreq); + } + if self.thermal_zone { + collectors.push(Collector::ThermalZone); + } + if self.nvme { + collectors.push(Collector::Nvme); + } + if self.hwmon { + collectors.push(Collector::Hwmon); + } + if self.time { + collectors.push(Collector::Time); + } + if self.textfile { + if self.textfile_directory.is_none() { + return Err( + "Textfile collector requires --node-metrics-receiver-textfile-directory".into(), + ); + } + collectors.push(Collector::Textfile); + } else if self.textfile_directory.is_some() { + tracing::warn!( + "--node-metrics-receiver-textfile-directory is set but textfile collector is disabled; \ + add --node-metrics-receiver-textfile true to enable it" + ); + } + + // When rootfs is set to something other than "/", compose it onto the + // procfs and sysfs paths. This allows containerized setups to set a single + // --node-metrics-receiver-rootfs-path=/host instead of separate + // --node-metrics-receiver-procfs-path=/host/proc and + // --node-metrics-receiver-sysfs-path=/host/sys. + // `trim_end_matches` already reduces "/" to "", so only the empty case remains. + let rootfs = self.rootfs_path.trim_end_matches('/'); + + // Reject the combination rather than silently producing "/host/host/proc", which + // would leave every collector reading a path that does not exist. + // Compare with path-component awareness: a plain `starts_with` would reject + // `--procfs-path /hostile/proc` against `--rootfs-path /host`, which is not + // prefixed at all and composes correctly. + if !rootfs.is_empty() + && (matches_path_prefix(&self.procfs_path, rootfs) + || matches_path_prefix(&self.sysfs_path, rootfs)) + { + return Err(format!( + "--node-metrics-receiver-rootfs-path ({}) is prefixed onto the procfs and sysfs \ + paths, so it cannot be combined with an explicit \ + --node-metrics-receiver-procfs-path ({}) or --node-metrics-receiver-sysfs-path \ + ({}). Set either the rootfs path or the individual paths.", + self.rootfs_path, self.procfs_path, self.sysfs_path + ) + .into()); + } + + let compose = |path: &str| -> String { + if rootfs.is_empty() { + path.to_string() + } else { + // Join on a single separator regardless of how the suffix is written + format!("{}/{}", rootfs, path.trim_start_matches('/')) + } + }; + + let procfs_path = compose(&self.procfs_path); + let sysfs_path = compose(&self.sysfs_path); + // Deliberately not composed: the textfile directory is rotel's own data, not a + // kernel pseudo-filesystem, and prefixing it would point at a path that usually + // does not exist inside the host mount. + let textfile_directory = self.textfile_directory.clone(); + + let mut config = NodeMetricsConfig { + scrape_interval: self.scrape_interval, + collectors, + metric_include_filter: None, + metric_exclude_filter: None, + filesystem_mount_exclude: None, + procfs_path, + sysfs_path, + rootfs_path: self.rootfs_path.clone(), + textfile_directory, + service_name: self.service_name.clone(), + }; + + // Set filters if provided + if let Some(ref pattern) = self.include_filter { + config = config.with_include_filter(pattern).map_err(|e| { + format!( + "Invalid --node-metrics-receiver-include-filter regex: {}", + e + ) + })?; + } + + if let Some(ref pattern) = self.exclude_filter { + config = config.with_exclude_filter(pattern).map_err(|e| { + format!( + "Invalid --node-metrics-receiver-exclude-filter regex: {}", + e + ) + })?; + } + + if let Some(ref pattern) = self.filesystem_mount_exclude { + config = config.with_filesystem_mount_exclude(pattern).map_err(|e| { + format!( + "Invalid --node-metrics-receiver-filesystem-mount-exclude regex: {}", + e + ) + })?; + } + + config.normalize_and_validate().map_err(|e| { + format!( + "{} (see --node-metrics-receiver-scrape-interval and the \ + --node-metrics-receiver- flags)", + e + ) + })?; + + Ok(config) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_args() { + let args = NodeMetricsReceiverArgs::default(); + + let config = args.build_config().unwrap(); + assert_eq!(config.scrape_interval, Duration::from_secs(60)); + // Compare the actual collector set: a length check would still pass if one + // collector were swapped for another. + assert_eq!(config.collectors, NodeMetricsConfig::default().collectors); + assert_eq!(config.procfs_path, DEFAULT_PROCFS_PATH); + assert_eq!(config.sysfs_path, DEFAULT_SYSFS_PATH); + } + + /// The hand-written `Default` impl duplicates ~28 clap `default_value` strings. + /// Without this test, changing one and forgetting the other ships a different + /// default than the CLI documents. + #[test] + fn test_clap_defaults_match_default_impl() { + use clap::Parser; + + /// Minimal command that flattens only these args, so clap can be driven with no + /// arguments at all + #[derive(Debug, Parser)] + struct OnlyNodeMetrics { + #[command(flatten)] + args: NodeMetricsReceiverArgs, + } + + let parsed = OnlyNodeMetrics::parse_from(["rotel"]).args; + let defaulted = NodeMetricsReceiverArgs::default(); + + let from_clap = parsed.build_config().unwrap(); + let from_default = defaulted.build_config().unwrap(); + + assert_eq!(from_clap.scrape_interval, from_default.scrape_interval); + assert_eq!(from_clap.collectors, from_default.collectors); + assert_eq!(from_clap.procfs_path, from_default.procfs_path); + assert_eq!(from_clap.sysfs_path, from_default.sysfs_path); + assert_eq!(from_clap.service_name, from_default.service_name); + assert_eq!( + from_clap.textfile_directory, + from_default.textfile_directory + ); + } + + /// The collector toggles must accept an explicit value; clap would otherwise infer + /// `SetTrue` from the `bool` type and make `--...-cpu false` a parse error. + #[test] + fn test_collector_toggles_accept_explicit_values() { + use clap::Parser; + + #[derive(Debug, Parser)] + struct OnlyNodeMetrics { + #[command(flatten)] + args: NodeMetricsReceiverArgs, + } + + let args = OnlyNodeMetrics::parse_from([ + "rotel", + "--node-metrics-receiver-cpu", + "false", + "--node-metrics-receiver-vmstat=false", + ]) + .args; + + assert!(!args.cpu); + assert!(!args.vmstat); + assert!(args.loadavg, "untouched toggles stay enabled"); + + let config = args.build_config().unwrap(); + assert!(!config.collectors.contains(&Collector::Cpu)); + assert!(!config.collectors.contains(&Collector::Vmstat)); + assert!(config.collectors.contains(&Collector::Loadavg)); + } + + #[test] + fn test_rootfs_path_conflicts_with_explicit_procfs_path() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + procfs_path: "/host/proc".to_string(), + ..Default::default() + }; + + // Silently composing these would yield /host/host/proc + let err = args.build_config().unwrap_err().to_string(); + assert!( + err.contains("--node-metrics-receiver-rootfs-path"), + "error should name the conflicting flags, got: {}", + err + ); + } + + #[test] + fn test_rootfs_path_composes_onto_defaults() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host/".to_string(), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert_eq!(config.procfs_path, "/host/proc"); + assert_eq!(config.sysfs_path, "/host/sys"); + } + + /// clap's duplicate-id and conflicting-flag assertions only fire when the whole + /// command is built, which no other test does. This makes the explicit `id(...)` + /// values self-enforcing against a future collision with another receiver's args. + #[test] + fn test_full_cli_definition_is_valid() { + use clap::{CommandFactory, Parser}; + + // Flattening every receiver's and exporter's args is what makes a duplicate id + // or flag collide, so build the whole agent command here. + #[derive(Debug, Parser)] + struct FullAgent { + #[command(flatten)] + agent: crate::init::args::AgentRun, + } + + FullAgent::command().debug_assert(); + } + + /// The toggles take a value, so a bare flag must be rejected rather than silently + /// meaning "true". + #[test] + fn test_bare_collector_toggle_is_an_error() { + use clap::Parser; + + #[derive(Debug, Parser)] + struct OnlyNodeMetrics { + #[command(flatten)] + args: NodeMetricsReceiverArgs, + } + + let result = OnlyNodeMetrics::try_parse_from(["rotel", "--node-metrics-receiver-cpu"]); + assert!( + result.is_err(), + "a bare collector toggle must require a value" + ); + } + + #[test] + fn test_scrape_interval_accepts_duration_strings() { + use clap::Parser; + + #[derive(Debug, Parser)] + struct OnlyNodeMetrics { + #[command(flatten)] + args: NodeMetricsReceiverArgs, + } + + let args = + OnlyNodeMetrics::parse_from(["rotel", "--node-metrics-receiver-scrape-interval", "2m"]) + .args; + assert_eq!(args.scrape_interval, Duration::from_secs(120)); + } + + #[test] + fn test_rootfs_path_conflicts_with_explicit_sysfs_path() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + sysfs_path: "/host/sys".to_string(), + ..Default::default() + }; + + let err = args.build_config().unwrap_err().to_string(); + assert!( + err.contains("--node-metrics-receiver-rootfs-path"), + "error should name the conflicting flags, got: {}", + err + ); + } + + /// A path whose first component merely *begins* with the rootfs string is not under it, + /// so it must compose rather than be rejected. A plain `starts_with` got this wrong. + #[test] + fn test_rootfs_path_accepts_a_path_that_only_shares_a_prefix_string() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + procfs_path: "/hostile/proc".to_string(), + ..Default::default() + }; + + let config = args + .build_config() + .expect("/hostile/proc is not under /host and must be accepted"); + assert_eq!(config.procfs_path, "/host/hostile/proc"); + } + + /// An explicit procfs path that is not under the rootfs prefix is a legitimate + /// combination and must still compose. + #[test] + fn test_rootfs_path_composes_onto_non_default_procfs_path() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + procfs_path: "/proc-alt".to_string(), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert_eq!(config.procfs_path, "/host/proc-alt"); + } + + /// The textfile directory is rotel's own data, so the rootfs prefix must not be + /// applied to it. + #[test] + fn test_rootfs_path_does_not_compose_onto_textfile_directory() { + let args = NodeMetricsReceiverArgs { + rootfs_path: "/host".to_string(), + textfile: true, + textfile_directory: Some("/var/lib/rotel/textfile".to_string()), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert_eq!( + config.textfile_directory.as_deref(), + Some("/var/lib/rotel/textfile") + ); + } + + /// The receiver layer reports the property; the flag name is added here. + #[test] + fn test_validation_error_points_at_the_flags() { + let args = NodeMetricsReceiverArgs { + scrape_interval: Duration::from_millis(500), + ..Default::default() + }; + + let err = args.build_config().unwrap_err().to_string(); + assert!( + err.contains("--node-metrics-receiver-"), + "error should point at the receiver's flags, got: {}", + err + ); + assert!( + err.contains("500ms"), + "error should report the value, got: {}", + err + ); + } + + #[test] + fn test_selective_collectors() { + let args = NodeMetricsReceiverArgs { + scrape_interval: Duration::from_secs(30), + cpu: false, + loadavg: true, + memory: true, + network: false, + filesystem: false, + uname: false, + stat: false, + processes: false, + diskstats: false, + vmstat: false, + netstat: false, + sockstat: false, + filefd: false, + cpufreq: false, + thermal_zone: false, + nvme: false, + hwmon: false, + time: false, + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert_eq!( + config.collectors, + vec![Collector::Loadavg, Collector::Memory] + ); + } + + #[test] + fn test_with_filters() { + let args = NodeMetricsReceiverArgs { + loadavg: true, + include_filter: Some("node_load.*".to_string()), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + // Assert the filter's effect, not just that one was compiled + assert!(config.should_include_metric("node_load1")); + assert!(!config.should_include_metric("node_cpu_seconds_total")); + } + + #[test] + fn test_no_collectors_error() { + let args = NodeMetricsReceiverArgs { + cpu: false, + loadavg: false, + memory: false, + network: false, + filesystem: false, + uname: false, + stat: false, + processes: false, + diskstats: false, + vmstat: false, + netstat: false, + sockstat: false, + filefd: false, + cpufreq: false, + thermal_zone: false, + nvme: false, + hwmon: false, + time: false, + ..Default::default() + }; + + let result = args.build_config(); + assert!(result.is_err()); + } + + #[test] + fn test_invalid_filter_regex() { + let args = NodeMetricsReceiverArgs { + include_filter: Some("[invalid".to_string()), + ..Default::default() + }; + + // The whole point of the wrapper is to name the flag + let err = args.build_config().unwrap_err().to_string(); + assert!( + err.contains("--node-metrics-receiver-include-filter"), + "error should name the offending flag, got: {}", + err + ); + } + + #[test] + fn test_filesystem_mount_exclude() { + let args = NodeMetricsReceiverArgs { + filesystem_mount_exclude: Some("^/nfs/".to_string()), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert!(config.filesystem_mount_exclude.is_some()); + let re = config.filesystem_mount_exclude.as_ref().unwrap(); + assert!(re.is_match("/nfs/share")); + assert!(!re.is_match("/mnt/data")); + } + + #[test] + fn test_textfile_requires_directory() { + let args = NodeMetricsReceiverArgs { + textfile: true, + textfile_directory: None, + ..Default::default() + }; + + let result = args.build_config(); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("textfile-directory") + ); + } + + #[test] + fn test_textfile_with_directory() { + let args = NodeMetricsReceiverArgs { + textfile: true, + textfile_directory: Some("/var/lib/node_exporter/textfile".to_string()), + ..Default::default() + }; + + let config = args.build_config().unwrap(); + assert!(config.collectors.contains(&Collector::Textfile)); + assert_eq!( + config.textfile_directory, + Some("/var/lib/node_exporter/textfile".to_string()) + ); + } +} diff --git a/src/receivers/mod.rs b/src/receivers/mod.rs index 56be3db0..7b3287f5 100644 --- a/src/receivers/mod.rs +++ b/src/receivers/mod.rs @@ -14,6 +14,9 @@ pub mod file; #[cfg(all(target_os = "linux", feature = "kmsg_receiver"))] pub mod kmsg; +#[cfg(feature = "node_metrics_receiver")] +pub mod node_metrics; + use opentelemetry::global; use opentelemetry::metrics::Meter; diff --git a/src/receivers/node_metrics/collector/mod.rs b/src/receivers/node_metrics/collector/mod.rs new file mode 100644 index 00000000..5bcbce99 --- /dev/null +++ b/src/receivers/node_metrics/collector/mod.rs @@ -0,0 +1,478 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! System metrics collector reading directly from /proc and /sys +//! +//! This collector reads Linux kernel interfaces, emitting raw counter values +//! compatible with Prometheus node_exporter conventions. + +// Notice: portions of this collector are derived from Prometheus node_exporter +// (https://github.com/prometheus/node_exporter), licensed under the Apache License 2.0. +/* Copyright © The Prometheus Authors */ +// +// Derived material, by submodule: +// procfs.rs metric names and HELP strings; the default filesystem-type and +// mount-point exclusion lists +// sysfs.rs metric names and HELP strings; the hwmon chip-naming approach from +// that project's `collector/hwmon.go` +// textfile.rs the textfile collector's behaviour and its `node_textfile_*` semantics +// util.rs the virtual-device and partition exclusion heuristics +// +// Where behaviour deliberately differs from node_exporter, the relevant function says so. + +mod procfs; +mod sysfs; +mod textfile; +mod util; + +pub(crate) use util::matches_path_prefix; + +#[cfg(test)] +mod test_support; + +use crate::receivers::node_metrics::config::{Collector, NodeMetricsConfig}; +use std::fs; +use std::time::{SystemTime, UNIX_EPOCH}; +// Use portable-atomic on platforms without native 64-bit atomics (e.g. 32-bit ARM) +#[cfg(not(target_has_atomic = "64"))] +use portable_atomic::{AtomicU64, Ordering}; +#[cfg(target_has_atomic = "64")] +use std::sync::atomic::{AtomicU64, Ordering}; +use tracing::{debug, info, warn}; + +/// Collected metric value with optional labels +/// +/// Metric names, units and descriptions are allocated afresh on every scrape. +#[derive(Debug, Clone)] +pub struct CollectedMetric { + pub name: String, + pub value: f64, + pub labels: Vec<(String, String)>, + pub metric_type: MetricType, + pub unit: Option, + pub description: Option, +} + +/// Type of metric (determines how it's represented in OTLP) +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum MetricType { + /// Gauge - current value that can go up or down + Gauge, + /// Counter - monotonically increasing value + Counter, +} + +/// System metrics collector reading from /proc and /sys +pub struct SystemCollector { + /// Clock ticks per second (for CPU time conversion) + clk_tck: f64, + /// Memory page size in bytes (for sockstat mem-to-bytes conversion) + page_size: f64, + /// System boot time in seconds since epoch, or 0 while it is unknown. + /// + /// Re-read on every scrape that reads `/proc/stat`, because the kernel's `btime` + /// moves whenever the wall clock is stepped — a device with no RTC boots near the + /// epoch and then jumps when NTP syncs, and a value latched before that would be + /// wrong forever. + boot_time: AtomicU64, + /// Boot time used as `start_time_unix_nano` for cumulative counters, or 0 while + /// unknown. + /// + /// Unlike `boot_time` this is latched at the first non-zero reading: the counters + /// themselves do not reset when the clock is stepped, and moving their start time + /// would be reported downstream as a counter reset. + /// + /// One consequence: if the clock is later stepped *backwards* past this value, counters + /// briefly carry a start time ahead of their own timestamp, which a strict backend may + /// reject. It resolves itself on the next forward correction, and the alternative — + /// re-latching — would signal a reset on every clock adjustment. + counter_start_time: AtomicU64, + /// Path to procfs mount point + procfs_path: String, + /// Path to sysfs mount point + sysfs_path: String, +} + +impl SystemCollector { + /// Create a new system collector with configurable procfs and sysfs paths + pub fn new(procfs_path: &str, sysfs_path: &str) -> Self { + // Get clock ticks per second for CPU time conversion. + // sysconf returns -1 on error; fall back to the POSIX-common default of 100. + let raw_clk = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; + let clk_tck = if raw_clk > 0 { raw_clk as f64 } else { 100.0 }; + // Get memory page size for sockstat mem-to-bytes conversion. + // sysconf returns -1 on error; fall back to the common default of 4096. + let raw_page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + let page_size = if raw_page > 0 { + raw_page as f64 + } else { + 4096.0 + }; + + // Get boot time from /proc/stat + let boot_time = Self::read_boot_time(procfs_path).unwrap_or_else(|| { + warn!( + "Could not read boot time from {}/stat; \ + counter start_time will default to epoch (0) until it can be read", + procfs_path + ); + 0 + }); + + Self { + clk_tck, + page_size, + boot_time: AtomicU64::new(boot_time), + counter_start_time: AtomicU64::new(boot_time), + procfs_path: procfs_path.to_string(), + sysfs_path: sysfs_path.to_string(), + } + } + + /// Read boot time from /proc/stat + fn read_boot_time(procfs_path: &str) -> Option { + let path = format!("{}/stat", procfs_path); + let content = fs::read_to_string(path).ok()?; + Self::parse_boot_time(&content) + } + + /// Extract the `btime` field from the contents of /proc/stat + fn parse_boot_time(stat_content: &str) -> Option { + for line in stat_content.lines() { + if let Some(rest) = line.strip_prefix("btime ") { + // The kernel formats btime from a *signed* timespec64 as %llu, so a machine + // whose wall clock is near the epoch prints a wrapped value (~1.8e19) after + // a backwards clock step or a suspend/resume. Accepting it would publish a + // nonsense boot time and, worse, latch it as every counter's start time. + let btime: i64 = rest.trim().parse().ok()?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(i64::MAX); + if btime <= 0 || btime > now { + warn!( + "Ignoring implausible boot time from /proc/stat: btime {}", + rest.trim() + ); + return None; + } + return Some(btime as u64); + } + } + None + } + + /// Get the current boot time in seconds since epoch, or 0 if it is not known + pub fn boot_time(&self) -> u64 { + self.boot_time.load(Ordering::Relaxed) + } + + /// Get the boot time to use as `start_time_unix_nano` for cumulative counters, or 0 + /// if it is not known. + /// + /// Latched at the first known value so that a wall-clock step does not look like a + /// counter reset downstream. + pub fn counter_start_time(&self) -> u64 { + self.counter_start_time.load(Ordering::Relaxed) + } + + /// Collect all enabled metrics based on config + pub fn collect(&self, config: &NodeMetricsConfig) -> Vec { + // A default collector set on a many-core host produces well over a thousand + // metrics (vmstat and netstat alone contribute several hundred). + let mut metrics = Vec::with_capacity(2048); + + // Read /proc/stat once for the collectors that need it and for the boot time. + // `Time` is included because `node_boot_time_seconds` comes from the `btime` line: + // without it, a clock step would go unnoticed whenever cpu and stat are disabled. + let stat_content = if self.counter_start_time() == 0 + || config + .collectors + .iter() + .any(|c| matches!(c, Collector::Cpu | Collector::Stat | Collector::Time)) + { + let path = format!("{}/stat", self.procfs_path); + match fs::read_to_string(&path) { + Ok(c) => Some(c), + Err(e) => { + debug!("Failed to read {}: {}", path, e); + None + } + } + } else { + None + }; + + // Refresh the boot time every scrape: the kernel's `btime` moves whenever the wall + // clock is stepped, and procfs may only have become readable after startup. + if let Some(btime) = stat_content.as_deref().and_then(Self::parse_boot_time) { + let previous = self.boot_time.swap(btime, Ordering::Relaxed); + // btime tracks the timekeeper's wall-clock offset, so it also drifts by a + // second under ordinary slewing, suspend/resume or VM migration. Only a real + // step is worth an info line; the rest is debug. + if previous.abs_diff(btime) > 1 { + info!("System boot time is {} (previously {})", btime, previous); + } else if previous != btime { + debug!("System boot time adjusted to {} (was {})", btime, previous); + } + // Latch the counter start time at the first known value, so a clock step is + // not reported downstream as a counter reset. + let _ = self.counter_start_time.compare_exchange( + 0, + btime, + Ordering::Relaxed, + Ordering::Relaxed, + ); + } + + for collector in &config.collectors { + let result = match collector { + Collector::Cpu => self.collect_cpu(config, stat_content.as_deref()), + Collector::Loadavg => self.collect_loadavg(config), + Collector::Memory => self.collect_meminfo(config), + Collector::Network => self.collect_netdev(config), + Collector::Filesystem => self.collect_filesystem(config), + Collector::Uname => self.collect_uname(config), + Collector::Stat => self.collect_stat(config, stat_content.as_deref()), + Collector::Processes => self.collect_processes(config), + Collector::Diskstats => self.collect_diskstats(config), + Collector::Vmstat => self.collect_vmstat(config), + Collector::Netstat => self.collect_netstat(config), + Collector::Sockstat => self.collect_sockstat(config), + Collector::Filefd => self.collect_filefd(config), + Collector::Cpufreq => self.collect_cpufreq(config), + Collector::ThermalZone => self.collect_thermal_zone(config), + Collector::Nvme => self.collect_nvme(config), + Collector::Hwmon => self.collect_hwmon(config), + Collector::Textfile => self.collect_textfile(config), + Collector::Time => self.collect_time(config), + }; + metrics.extend(result); + } + + metrics + } +} + +impl Default for SystemCollector { + fn default() -> Self { + Self::new("/proc", "/sys") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::receivers::node_metrics::collector::test_support::*; + + // --------------------------------------------------------------------- + // Construction + // --------------------------------------------------------------------- + + #[test] + fn test_boot_time_read_from_stat() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "stat", STAT_FIXTURE); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + assert_eq!(collector.boot_time(), 1_700_000_000); + + // Without a readable /proc/stat the boot time stays unknown rather than + // being invented; collect() retries the read on every scrape. + let empty = tempfile::tempdir().unwrap(); + let without_stat = SystemCollector::new(empty.path().to_str().unwrap(), "/sys"); + assert_eq!(without_stat.boot_time(), 0); + } + + #[test] + fn test_boot_time_retried_on_collect_until_readable() { + // A procfs that is not mounted yet when the receiver starts: without the + // per-scrape retry every counter would keep epoch 0 as its start time for + // the whole life of the process. + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + let collector = SystemCollector::new(&path, &path); + assert_eq!(collector.boot_time(), 0); + + // While boot time is unknown the time collector suppresses the metric + // rather than reporting a plausible-looking 1970 boot. + let before = collector.collect(&config_for(vec![Collector::Time])); + assert!(matching(&before, "node_boot_time_seconds", &[]).is_empty()); + find_one(&before, "node_time_seconds", &[]); + + // procfs appears between two scrapes. + write(tmp.path(), "stat", STAT_FIXTURE); + let after = collector.collect(&config_for(vec![Collector::Time])); + + assert_eq!(collector.boot_time(), 1_700_000_000); + let boot = find_one(&after, "node_boot_time_seconds", &[]); + assert_eq!(boot.value, 1_700_000_000.0); + assert_eq!(boot.unit.as_deref(), Some("s")); + assert_eq!(boot.metric_type, MetricType::Gauge); + assert!(boot.labels.is_empty()); + } + + /// The kernel formats `btime` from a signed value as `%llu`, so a machine whose clock + /// is near the epoch can print a wrapped value. Accepting it would publish a boot time + /// ~580 billion years in the future and latch it into every counter's start time. + #[test] + fn test_implausible_boot_time_is_rejected() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + // 2^64 - 10, i.e. -10 seconds printed as unsigned + write( + tmp.path(), + "stat", + "btime 18446744073709551606\ncpu 1 2 3 4 5 6 7 8 9 10\n", + ); + + let collector = SystemCollector::new(&path, &path); + assert_eq!(collector.boot_time(), 0, "a wrapped btime must be rejected"); + assert_eq!(collector.counter_start_time(), 0); + + // A boot time in the future is equally implausible + write(tmp.path(), "stat", "btime 99999999999\n"); + let metrics = collector.collect(&config_for(vec![Collector::Time])); + assert_eq!(collector.boot_time(), 0); + assert!(matching(&metrics, "node_boot_time_seconds", &[]).is_empty()); + + // ...and a plausible one is still accepted + write(tmp.path(), "stat", STAT_FIXTURE); + collector.collect(&config_for(vec![Collector::Time])); + assert_eq!(collector.boot_time(), 1_700_000_000); + } + + /// A wall-clock step moves the kernel's `btime`, so the reported boot time must track + /// it — while the start time stamped on cumulative counters stays put, since moving it + /// would be read downstream as a counter reset. + #[test] + fn test_boot_time_tracks_clock_steps_but_counter_start_time_is_latched() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + write(tmp.path(), "stat", STAT_FIXTURE); + + let collector = SystemCollector::new(&path, &path); + assert_eq!(collector.boot_time(), 1_700_000_000); + assert_eq!(collector.counter_start_time(), 1_700_000_000); + + // The clock is stepped forward, so the kernel reports a later boot time. This is + // what an RTC-less device does when NTP first syncs. + write( + tmp.path(), + "stat", + &STAT_FIXTURE.replace("btime 1700000000", "btime 1700000500"), + ); + let after = collector.collect(&config_for(vec![Collector::Time])); + + // The gauge follows the kernel... + assert_eq!(collector.boot_time(), 1_700_000_500); + assert_eq!( + find_one(&after, "node_boot_time_seconds", &[]).value, + 1_700_000_500.0 + ); + // ...but the counter start time keeps its first known value. + assert_eq!(collector.counter_start_time(), 1_700_000_000); + } + + /// `/proc/stat` must be read when the `time` collector is enabled, even with cpu and + /// stat disabled and a boot time already latched — otherwise a clock step goes unseen. + #[test] + fn test_time_collector_alone_still_refreshes_boot_time() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().to_str().unwrap().to_string(); + write(tmp.path(), "stat", STAT_FIXTURE); + + let collector = SystemCollector::new(&path, &path); + assert_eq!(collector.counter_start_time(), 1_700_000_000); + + write( + tmp.path(), + "stat", + &STAT_FIXTURE.replace("btime 1700000000", "btime 1700009999"), + ); + // Only Time — neither Cpu nor Stat forces the /proc/stat read here. + collector.collect(&config_for(vec![Collector::Time])); + + assert_eq!(collector.boot_time(), 1_700_009_999); + } + + // --------------------------------------------------------------------- + // Top-level dispatch + // --------------------------------------------------------------------- + + #[test] + fn test_collect_with_filter() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "loadavg", "0.52 0.58 0.59 2/1234 5678\n"); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let config = NodeMetricsConfig::new(60) + .with_include_filter("^node_load1$") + .unwrap(); + + let metrics = collector.collect_loadavg(&config); + + assert_eq!(metrics.len(), 1); + assert_eq!(metrics[0].name, "node_load1"); + assert_eq!(metrics[0].value, 0.52); + } + + #[test] + fn test_collect_dispatches_to_every_enabled_collector() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "stat", STAT_FIXTURE); + + let path = tmp.path().to_str().unwrap(); + let collector = SystemCollector::new(path, path); + let metrics = collector.collect(&config_for(vec![Collector::Cpu, Collector::Stat])); + + // A single collect() call returns the metrics of both collectors, from one + // read of /proc/stat. + assert_eq!( + find_one( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "0"), ("mode", "user")] + ) + .value, + 1000.0 / collector.clk_tck + ); + assert_eq!(find_one(&metrics, "node_forks_total", &[]).value, 4242.0); + } + + #[test] + fn test_collect_without_stat_file() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "loadavg", "1.5 1.0 0.5 1/100 200\n"); + + let path = tmp.path().to_str().unwrap(); + let collector = SystemCollector::new(path, path); + let metrics = collector.collect(&config_for(vec![Collector::Loadavg])); + + assert_eq!(metrics.len(), 3); + assert_eq!(find_one(&metrics, "node_load1", &[]).value, 1.5); + // The boot-time retry finds nothing to read and leaves it unknown. + assert_eq!(collector.boot_time(), 0); + } + + #[test] + fn test_collect_empty_fixture_reports_only_syscall_backed_metrics() { + let procfs = tempfile::tempdir().unwrap(); + let sysfs = tempfile::tempdir().unwrap(); + + let mut collectors = NodeMetricsConfig::default().collectors; + collectors.push(Collector::Textfile); + + let collector = SystemCollector::new( + procfs.path().to_str().unwrap(), + sysfs.path().to_str().unwrap(), + ); + let metrics = collector.collect(&config_for(collectors)); + + // Every file-backed collector finds nothing and reports nothing, without + // panicking. Only uname and the wall clock, which read no files, remain; + // node_boot_time_seconds is suppressed because btime is unknown. + let mut names: Vec<&str> = metrics.iter().map(|m| m.name.as_str()).collect(); + names.sort_unstable(); + assert_eq!(names, ["node_time_seconds", "node_uname_info"]); + } +} diff --git a/src/receivers/node_metrics/collector/procfs.rs b/src/receivers/node_metrics/collector/procfs.rs new file mode 100644 index 00000000..be740985 --- /dev/null +++ b/src/receivers/node_metrics/collector/procfs.rs @@ -0,0 +1,2575 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Collectors reading `/proc`: CPU, load average, memory, network, kernel stat counters, +//! processes, disk I/O, vmstat, netstat and socket/file-descriptor statistics. +//! +//! Three members are wholly or partly syscall-backed but live here because they belong to +//! the same host-level group: `collect_uname` calls `uname(2)`, `collect_time` reads the +//! wall clock, and `collect_filesystem` takes the mount list from `/proc/mounts` but its +//! values from `statfs(2)`. +//! +//! Metric names and HELP strings, and the filesystem exclusion lists, are derived from +//! Prometheus node_exporter — see the notice in this module's `mod.rs`. + +use crate::receivers::node_metrics::collector::util::{ + decode_mount_path, is_partition, is_virtual_disk, matches_path_prefix, utsname_field, +}; +use crate::receivers::node_metrics::collector::{CollectedMetric, MetricType, SystemCollector}; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use std::fs; +use tracing::{debug, warn}; + +// On Linux use the LFS variant: plain `statfs` is the 32-bit interface on glibc and returns +// EOVERFLOW for a filesystem whose block or inode counts do not fit, which would drop large +// filesystems entirely on the 32-bit targets this receiver supports. The BSD `statfs` is +// already 64-bit. +#[cfg(not(target_os = "linux"))] +use libc::{statfs, statfs as StatFs}; +#[cfg(target_os = "linux")] +use libc::{statfs64 as StatFs, statfs64 as statfs}; + +impl SystemCollector { + /// Collect CPU metrics from /proc/stat + /// Emits `node_cpu_seconds_total` (modes: user..steal) and + /// `node_cpu_guest_seconds_total` (modes: user, nice), matching + /// Prometheus node_exporter conventions. + pub(super) fn collect_cpu( + &self, + config: &NodeMetricsConfig, + stat_content: Option<&str>, + ) -> Vec { + let mut metrics = Vec::new(); + + let content = match stat_content { + Some(c) => c, + None => return metrics, + }; + + // /proc/stat fields (indices 1..=10): + // user nice system idle iowait irq softirq steal guest guest_nice + // Fields 1-8 -> node_cpu_seconds_total + // Fields 9-10 -> node_cpu_guest_seconds_total (modes: user, nice) + let cpu_modes = [ + "user", "nice", "system", "idle", "iowait", "irq", "softirq", "steal", + ]; + let guest_modes = ["user", "nice"]; + + for line in content.lines() { + // Match cpu lines (cpu, cpu0, cpu1, etc.) + if !line.starts_with("cpu") { + continue; + } + + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.is_empty() { + continue; + } + + let cpu_label = parts[0]; + + // Skip the aggregate "cpu" line — only report per-core metrics, + // matching Prometheus node_exporter behaviour. + if cpu_label == "cpu" { + continue; + } + let cpu_num = if let Some(num) = cpu_label.strip_prefix("cpu") { + num.to_string() + } else { + continue; + }; + + // node_cpu_seconds_total: fields 1-8 + if config.should_include_metric("node_cpu_seconds_total") { + for (i, mode) in cpu_modes.iter().enumerate() { + let field_idx = i + 1; // user=1, nice=2, ... softirq=7, steal=8 + if field_idx >= parts.len() { + break; + } + // Skip rather than substitute 0: for a monotonic counter a fabricated + // zero looks like a counter reset and makes rate() spike. + let Ok(ticks) = parts[field_idx].parse::() else { + continue; + }; + let seconds = ticks / self.clk_tck; + + metrics.push(CollectedMetric { + name: "node_cpu_seconds_total".to_string(), + value: seconds, + labels: vec![ + ("cpu".to_string(), cpu_num.clone()), + ("mode".to_string(), mode.to_string()), + ], + metric_type: MetricType::Counter, + unit: Some("s".to_string()), + description: Some("Seconds the CPUs spent in each mode".to_string()), + }); + } + } + + // node_cpu_guest_seconds_total: fields 9-10 + if config.should_include_metric("node_cpu_guest_seconds_total") { + for (i, mode) in guest_modes.iter().enumerate() { + let field_idx = 9 + i; // guest=9, guest_nice=10 + if field_idx >= parts.len() { + break; + } + // Skip rather than substitute 0: for a monotonic counter a fabricated + // zero looks like a counter reset and makes rate() spike. + let Ok(ticks) = parts[field_idx].parse::() else { + continue; + }; + let seconds = ticks / self.clk_tck; + + metrics.push(CollectedMetric { + name: "node_cpu_guest_seconds_total".to_string(), + value: seconds, + labels: vec![ + ("cpu".to_string(), cpu_num.clone()), + ("mode".to_string(), mode.to_string()), + ], + metric_type: MetricType::Counter, + unit: Some("s".to_string()), + description: Some("Seconds the CPUs spent in guest mode".to_string()), + }); + } + } + } + + metrics + } + + /// Collect load average from /proc/loadavg + pub(super) fn collect_loadavg(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/loadavg", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + let parts: Vec<&str> = content.split_whitespace().collect(); + if parts.len() < 3 { + return metrics; + } + + let loads = [ + ("node_load1", parts[0], "1m load average"), + ("node_load5", parts[1], "5m load average"), + ("node_load15", parts[2], "15m load average"), + ]; + + for (name, value_str, description) in loads { + if config.should_include_metric(name) + && let Ok(value) = value_str.parse::() + { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some(description.to_string()), + }); + } + } + + metrics + } + + /// Collect memory metrics from /proc/meminfo + /// Dynamically reads ALL fields, converting kB to bytes + pub(super) fn collect_meminfo(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/meminfo", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + for line in content.lines() { + // Format: "FieldName: 12345 kB" or "FieldName: 12345" + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 2 { + continue; + } + + // Normalize parenthesized suffixes: "Active(anon)" -> "Active_anon" + let field_name = parts[0] + .trim_end_matches(':') + .replace('(', "_") + .replace(')', ""); + let value: f64 = match parts[1].parse() { + Ok(v) => v, + Err(_) => continue, + }; + + // Check if value is in kB (most fields are) + let (final_value, has_unit) = if parts.len() >= 3 && parts[2] == "kB" { + (value * 1024.0, true) // Convert kB to bytes + } else { + (value, false) + }; + + // Convert field name to metric name: "MemTotal" -> "node_memory_MemTotal_bytes" + let metric_name = if has_unit { + format!("node_memory_{}_bytes", field_name) + } else { + format!("node_memory_{}", field_name) + }; + + if config.should_include_metric(&metric_name) { + metrics.push(CollectedMetric { + name: metric_name, + value: final_value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: if has_unit { + Some("By".to_string()) + } else { + None + }, + description: None, + }); + } + } + + metrics + } + + /// Collect network device metrics from /proc/net/dev + pub(super) fn collect_netdev(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/net/dev", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + // Skip 2 header lines: + // "Inter-| Receive ..." + // " face |bytes packets ..." + for line in content.lines().skip(2) { + // Split on ':' first to cleanly separate interface name from values. + // The kernel format is "%6s: %8lu ..." so the colon is always present. + let (iface, rest) = match line.split_once(':') { + Some(pair) => pair, + None => continue, + }; + let interface = iface.trim(); + + let parts: Vec<&str> = rest.split_whitespace().collect(); + if parts.len() < 16 { + continue; + } + + let labels = vec![("device".to_string(), interface.to_string())]; + + // Receive fields (columns 0-7): bytes, packets, errs, drop, fifo, frame, compressed, multicast + // Transmit fields (columns 8-15): bytes, packets, errs, drop, fifo, colls, carrier, compressed + let netdev_metrics = [ + ( + "node_network_receive_bytes_total", + parts[0], + "By", + "Network device statistic receive_bytes", + ), + ( + "node_network_receive_packets_total", + parts[1], + "{packets}", + "Network device statistic receive_packets", + ), + ( + "node_network_receive_errs_total", + parts[2], + "{errors}", + "Network device statistic receive_errs", + ), + ( + "node_network_receive_drop_total", + parts[3], + "{packets}", + "Network device statistic receive_drop", + ), + ( + "node_network_receive_fifo_total", + parts[4], + "{events}", + "Network device statistic receive_fifo", + ), + ( + "node_network_receive_frame_total", + parts[5], + "{events}", + "Network device statistic receive_frame", + ), + ( + "node_network_receive_compressed_total", + parts[6], + "{packets}", + "Network device statistic receive_compressed", + ), + ( + "node_network_receive_multicast_total", + parts[7], + "{packets}", + "Network device statistic receive_multicast", + ), + ( + "node_network_transmit_bytes_total", + parts[8], + "By", + "Network device statistic transmit_bytes", + ), + ( + "node_network_transmit_packets_total", + parts[9], + "{packets}", + "Network device statistic transmit_packets", + ), + ( + "node_network_transmit_errs_total", + parts[10], + "{errors}", + "Network device statistic transmit_errs", + ), + ( + "node_network_transmit_drop_total", + parts[11], + "{packets}", + "Network device statistic transmit_drop", + ), + ( + "node_network_transmit_fifo_total", + parts[12], + "{events}", + "Network device statistic transmit_fifo", + ), + ( + "node_network_transmit_colls_total", + parts[13], + "{events}", + "Network device statistic transmit_colls", + ), + ( + "node_network_transmit_carrier_total", + parts[14], + "{events}", + "Network device statistic transmit_carrier", + ), + ( + "node_network_transmit_compressed_total", + parts[15], + "{packets}", + "Network device statistic transmit_compressed", + ), + ]; + + for (name, value_str, unit, description) in netdev_metrics { + if config.should_include_metric(name) + && let Ok(value) = value_str.parse::() + { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type: MetricType::Counter, + unit: Some(unit.to_string()), + description: Some(description.to_string()), + }); + } + } + } + + metrics + } + + /// Collect filesystem metrics using statfs + pub(super) fn collect_filesystem(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Prefer PID 1's mount table, as node_exporter does. `{procfs}/mounts` is a symlink + // to `self/mounts`, so with a bind-mounted host procfs it still yields *this* + // process's mount namespace — i.e. the container's — which would report the + // container's bind mounts labelled as if they were the host's filesystems. + let init_path = format!("{}/1/mounts", self.procfs_path); + let self_path = format!("{}/mounts", self.procfs_path); + let (path, content) = match fs::read_to_string(&init_path) { + Ok(c) => (init_path, c), + Err(e) => { + debug!( + "Failed to read {}: {}; falling back to {}", + init_path, e, self_path + ); + match fs::read_to_string(&self_path) { + Ok(c) => (self_path, c), + Err(e) => { + debug!("Failed to read {}: {}", self_path, e); + return metrics; + } + } + } + }; + debug!("Read mount table from {}", path); + + // Filesystem types to ignore (virtual filesystems) + let ignore_fstypes = [ + "sysfs", + "proc", + "devtmpfs", + "devpts", + "securityfs", + "cgroup", + "cgroup2", + "pstore", + "debugfs", + "hugetlbfs", + "mqueue", + "configfs", + "binfmt_misc", + "fusectl", + "fuse.lxcfs", // virtual, checked before the remote list + "overlay", + "nsfs", + "squashfs", + "autofs", + "tracefs", + "bpf", + "rpc_pipefs", + "selinuxfs", + "iso9660", + "procfs", // BSD-style procfs type name + ]; + + // Network filesystem types are skipped by default: `statfs` on a mount whose + // server has gone away blocks in the kernel uninterruptibly, and the scrape + // runs on a shared blocking thread that cannot be aborted. Local mounts can + // still be excluded individually with `filesystem_mount_exclude`. + let ignore_remote_fstypes = [ + "nfs", + "nfs4", + "cifs", + "smb3", + "smbfs", + "glusterfs", + "ceph", + "lustre", + "beegfs", + "gpfs", + "davfs", + "davfs2", + "9p", + "afs", + "ncpfs", + "orangefs", + "coda", + // Shared-disk cluster filesystems rather than network mounts, but excluded for + // the same reason: cluster locking can make statfs block indefinitely. + "ocfs2", + "gfs2", + // FUSE mounts are named by subtype; only the network-backed ones are + // excluded, so local FUSE filesystems (mergerfs, gocryptfs, encfs, bindfs) + // keep reporting. + "fuse.sshfs", + "fuse.s3fs", + "fuse.rclone", + "fuse.glusterfs", + "fuse.cephfs", + "fuse.davfs2", + "fuse.smbnetfs", + "fuse.curlftpfs", + "fuse.juicefs", + "fuse.gcsfuse", + "fuse.blobfuse2", + "fuse.goofys", + "fuse.s3ql", + "fuse.ceph-fuse", + "fuse.mfs", + ]; + + // Mount points to ignore (matched with path-boundary awareness: + // a prefix matches if the mount point equals it exactly or continues + // with '/', preventing "/dev" from matching "/developer"). + // Excluded including the mount point itself: these are kernel pseudo-filesystems. + let ignore_mount_paths = ["/proc", "/sys", "/dev"]; + // Excluded only *below* the path: a filesystem mounted exactly here is a real + // volume worth reporting (a dedicated /var/lib/docker disk is precisely the one + // most likely to fill up). This follows node_exporter for the docker and + // credentials paths, and is deliberately broader for /var/lib/containers (where it + // excludes only .../storage) and /run/user (which it does not exclude at all). + let ignore_mount_subpaths = [ + "/run/credentials", + "/run/user", + "/var/lib/docker", + "/var/lib/containers", + ]; + + // Deduplicate by (device, mountpoint), matching Prometheus node_exporter. + // /proc/mounts can contain duplicate entries from bind mounts or mount propagation. + let mut seen = std::collections::HashSet::new(); + + for line in content.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 4 { + continue; + } + + let device = decode_mount_path(parts[0]); + let mount_point = decode_mount_path(parts[1]); + let fstype = parts[2]; + // The mount options carry the read-only state, so it does not need a second + // syscall — and `statfs.f_flags` does not exist on Linux, only on the BSDs. + let read_only = parts[3].split(',').any(|opt| opt == "ro"); + + if !seen.insert((device.clone(), mount_point.clone())) { + continue; + } + + // Skip ignored filesystem types + if ignore_fstypes.contains(&fstype) { + continue; + } + + // Skip network filesystems, where statfs can block indefinitely + if ignore_remote_fstypes.contains(&fstype) { + debug!( + "Skipping remote filesystem {} of type {}", + mount_point, fstype + ); + continue; + } + + // Skip ignored mount point prefixes (with path boundary check: + // "/dev" matches "/dev" and "/dev/shm" but not "/developer") + if ignore_mount_paths + .iter() + .any(|prefix| matches_path_prefix(&mount_point, prefix)) + { + continue; + } + + if ignore_mount_subpaths + .iter() + .any(|prefix| mount_point != *prefix && matches_path_prefix(&mount_point, prefix)) + { + continue; + } + + // Skip mount points matching the user-provided exclude regex + // (useful for excluding NFS/CIFS mounts where statfs may hang) + if let Some(ref exclude) = config.filesystem_mount_exclude + && exclude.is_match(&mount_point) + { + continue; + } + + // Call statfs to get filesystem stats (matches Prometheus node_exporter) + let mut stat: StatFs = unsafe { std::mem::zeroed() }; + // `statfs` resolves in this process's mount namespace, so when the host root is + // reachable under a prefix the mount point has to be composed onto it. The + // label keeps the host's own path, so series match a host-side node_exporter. + let rootfs = config.rootfs_path.trim_end_matches('/'); + let statfs_target = if rootfs.is_empty() { + mount_point.clone() + } else { + format!("{}/{}", rootfs, mount_point.trim_start_matches('/')) + }; + + let mount_point_cstr = match std::ffi::CString::new(statfs_target.as_str()) { + Ok(c) => c, + Err(_) => continue, + }; + + let ret = unsafe { statfs(mount_point_cstr.as_ptr(), &mut stat) }; + let statfs_err = std::io::Error::last_os_error(); + if ret != 0 { + // Warn rather than debug: a filesystem that silently disappears from the + // metrics is what an operator needs to know about during an incident, and + // the usual causes (a stale mount, a permission) are actionable. + warn!("statfs({}) failed: {}", statfs_target, statfs_err); + continue; + } + + let labels = vec![ + ("device".to_string(), device), + ("fstype".to_string(), fstype.to_string()), + ("mountpoint".to_string(), mount_point), + ]; + + let block_size = stat.f_bsize as f64; + let total_bytes = stat.f_blocks as f64 * block_size; + let free_bytes = stat.f_bfree as f64 * block_size; + let avail_bytes = stat.f_bavail as f64 * block_size; + + let fs_metrics = [ + ( + "node_filesystem_size_bytes", + total_bytes, + "Filesystem size in bytes", + ), + ( + "node_filesystem_free_bytes", + free_bytes, + "Filesystem free space in bytes", + ), + ( + "node_filesystem_avail_bytes", + avail_bytes, + "Filesystem space available to non-root users in bytes", + ), + ]; + + for (name, value, description) in fs_metrics { + if config.should_include_metric(name) { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: Some("By".to_string()), + description: Some(description.to_string()), + }); + } + } + + // Files (inodes) + if config.should_include_metric("node_filesystem_files") { + metrics.push(CollectedMetric { + name: "node_filesystem_files".to_string(), + value: stat.f_files as f64, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Filesystem total file nodes".to_string()), + }); + } + + if config.should_include_metric("node_filesystem_files_free") { + metrics.push(CollectedMetric { + name: "node_filesystem_files_free".to_string(), + value: stat.f_ffree as f64, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Filesystem free file nodes".to_string()), + }); + } + + // Readonly flag, taken from the mount options above + if config.should_include_metric("node_filesystem_readonly") { + metrics.push(CollectedMetric { + name: "node_filesystem_readonly".to_string(), + value: if read_only { 1.0 } else { 0.0 }, + labels, + metric_type: MetricType::Gauge, + unit: None, + description: Some("Filesystem read-only status".to_string()), + }); + } + } + + metrics + } + + /// Collect uname/system information + pub(super) fn collect_uname(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Get uname info + if !config.should_include_metric("node_uname_info") { + return metrics; + } + + let mut uname_info: libc::utsname = unsafe { std::mem::zeroed() }; + let ret = unsafe { libc::uname(&mut uname_info) }; + + if ret == 0 { + // Convert C strings to Rust strings + let sysname = utsname_field(&uname_info.sysname); + let nodename = utsname_field(&uname_info.nodename); + let release = utsname_field(&uname_info.release); + let version = utsname_field(&uname_info.version); + let machine = utsname_field(&uname_info.machine); + + #[cfg(target_os = "linux")] + let domainname = utsname_field(&uname_info.domainname); + + #[allow(unused_mut)] // domainname is pushed only on Linux + let mut labels = vec![ + ("sysname".to_string(), sysname), + ("nodename".to_string(), nodename), + ("release".to_string(), release), + ("version".to_string(), version), + ("machine".to_string(), machine), + ]; + #[cfg(target_os = "linux")] + labels.push(("domainname".to_string(), domainname)); + + metrics.push(CollectedMetric { + name: "node_uname_info".to_string(), + value: 1.0, + labels, + metric_type: MetricType::Gauge, + unit: None, + description: Some( + "Labeled system information as provided by the uname system call".to_string(), + ), + }); + } + + metrics + } + + /// Collect time metrics (boot time, current time) + /// + /// This is a separate collector from uname, matching Prometheus node_exporter's + /// `time` collector. Boot time originates from `/proc/stat` (the `btime` line) and is + /// grouped here so it can be toggled independently. Note node_exporter emits + /// `node_boot_time_seconds` from its `stat` collector instead. + pub(super) fn collect_time(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Boot time, read from /proc/stat btime. Suppressed entirely while unknown: + // reporting 0 would look like a plausible (1970) boot time. + let boot_time = self.boot_time(); + if boot_time > 0 && config.should_include_metric("node_boot_time_seconds") { + metrics.push(CollectedMetric { + name: "node_boot_time_seconds".to_string(), + value: boot_time as f64, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("s".to_string()), + description: Some("Node boot time in seconds since epoch".to_string()), + }); + } + + // Current time. This reads the clock independently of the OTLP + // data-point timestamp set later in convert_to_otlp_metrics, so the + // two may differ by the time it takes to run the remaining collectors. + // Prometheus node_exporter has the same inherent skew. + if config.should_include_metric("node_time_seconds") { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or(0.0); + + metrics.push(CollectedMetric { + name: "node_time_seconds".to_string(), + value: now, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("s".to_string()), + description: Some("System time in seconds since epoch".to_string()), + }); + } + + metrics + } + + /// Collect kernel stat counters from /proc/stat + /// + /// Includes forks, context switches, interrupts, and procs running/blocked. + /// The per-CPU `cpu*` lines are handled by the cpu collector; `softirq` and the + /// per-interrupt vector of `intr` are not currently exposed. + pub(super) fn collect_stat( + &self, + config: &NodeMetricsConfig, + stat_content: Option<&str>, + ) -> Vec { + let mut metrics = Vec::new(); + + let content = match stat_content { + Some(c) => c, + None => return metrics, + }; + + for line in content.lines() { + if let Some(rest) = line.strip_prefix("processes ") { + if config.should_include_metric("node_forks_total") + && let Ok(value) = rest.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_forks_total".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Counter, + unit: None, + description: Some("Total number of forks".to_string()), + }); + } + } else if let Some(rest) = line.strip_prefix("ctxt ") { + if config.should_include_metric("node_context_switches_total") + && let Ok(value) = rest.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_context_switches_total".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Counter, + unit: None, + description: Some("Total number of context switches".to_string()), + }); + } + } else if let Some(rest) = line.strip_prefix("intr ") { + // Test the metric filter inside the arm, like the sibling arms do, so that + // control flow through this chain does not depend on the filter config. + // The first number is total interrupts. Skip the metric entirely if it + // cannot be parsed — a fabricated 0 reads as a counter reset downstream. + if config.should_include_metric("node_intr_total") + && let Some(total) = rest + .split_whitespace() + .next() + .and_then(|s| s.parse::().ok()) + { + metrics.push(CollectedMetric { + name: "node_intr_total".to_string(), + value: total, + labels: Vec::new(), + metric_type: MetricType::Counter, + unit: Some("{interrupts}".to_string()), + description: Some("Total number of interrupts serviced".to_string()), + }); + } + } else if let Some(rest) = line.strip_prefix("procs_running ") { + if config.should_include_metric("node_procs_running") + && let Ok(value) = rest.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_procs_running".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Number of processes in runnable state".to_string()), + }); + } + } else if let Some(rest) = line.strip_prefix("procs_blocked ") { + // Collapsing this into the chain condition would make control flow through + // the whole chain depend on the metric filter; only the final arm trips the + // lint, and all five arms deliberately test the filter inside the arm. + #[allow(clippy::collapsible_if)] + if config.should_include_metric("node_procs_blocked") + && let Ok(value) = rest.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_procs_blocked".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some( + "Number of processes blocked waiting for I/O".to_string(), + ), + }); + } + } + } + + metrics + } + + /// Collect process metrics: kernel limits from /proc/sys/kernel/ + pub(super) fn collect_processes(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Kernel limits (single-file reads, no per-PID scanning) + if config.should_include_metric("node_processes_max_threads") { + let path = format!("{}/sys/kernel/threads-max", self.procfs_path); + if let Ok(content) = fs::read_to_string(&path) + && let Ok(value) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_processes_max_threads".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Limit of threads in the system".to_string()), + }); + } + } + + if config.should_include_metric("node_processes_max_processes") { + let path = format!("{}/sys/kernel/pid_max", self.procfs_path); + if let Ok(content) = fs::read_to_string(&path) + && let Ok(value) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_processes_max_processes".to_string(), + value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: Some("Limit of PID values in the system".to_string()), + }); + } + } + + metrics + } + + /// Collect disk I/O statistics from /proc/diskstats + pub(super) fn collect_diskstats(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/diskstats", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + for line in content.lines() { + let fields: Vec<&str> = line.split_whitespace().collect(); + if fields.len() < 14 { + continue; + } + + let device = fields[2]; + + // Skip virtual devices, matching node_exporter's default exclusion of + // `^(z?ram|loop|fd)\d+$` — anchored on a trailing device number so that a + // real disk sharing one of these prefixes is not dropped. + if is_virtual_disk(device) { + continue; + } + + // Skip partitions: sda1, nvme0n1p1, etc. + // Keep whole disks: sda, nvme0n1, vda, xvda + if is_partition(device) { + continue; + } + + let labels = vec![("device".to_string(), device.to_string())]; + + // Parse helper: any field that cannot be parsed causes the entire device to + // be skipped, rather than emitting a misleading zero value or a partial set + // of metrics for that device. + let parse = |idx: usize| -> Option { fields[idx].parse().ok() }; + + // Parse core values - skip entire device on any parse error + let Some(rd_ios) = parse(3) else { continue }; + let Some(rd_merges) = parse(4) else { continue }; + let Some(rd_sectors) = parse(5) else { continue }; + let Some(rd_ticks) = parse(6) else { continue }; + let Some(wr_ios) = parse(7) else { continue }; + let Some(wr_merges) = parse(8) else { continue }; + let Some(wr_sectors) = parse(9) else { continue }; + let Some(wr_ticks) = parse(10) else { continue }; + let Some(ios_in_progress) = parse(11) else { + continue; + }; + let Some(io_ticks) = parse(12) else { continue }; + let Some(time_in_queue) = parse(13) else { + continue; + }; + + // Parse the optional extended field groups up front, so that a corrupt field + // skips the whole device instead of emitting the core metrics and then + // silently dropping the rest of the groups. + let discard = if fields.len() >= 18 { + match (parse(14), parse(15), parse(16), parse(17)) { + (Some(a), Some(b), Some(c), Some(d)) => Some((a, b, c, d)), + _ => continue, + } + } else { + None + }; + let flush = if fields.len() >= 20 { + match (parse(18), parse(19)) { + (Some(a), Some(b)) => Some((a, b)), + _ => continue, + } + } else { + None + }; + + // Convert sectors to bytes (sector = 512 bytes) + let rd_bytes = rd_sectors * 512.0; + let wr_bytes = wr_sectors * 512.0; + + // Convert milliseconds to seconds + let rd_time_seconds = rd_ticks / 1000.0; + let wr_time_seconds = wr_ticks / 1000.0; + let io_time_seconds = io_ticks / 1000.0; + let time_in_queue_seconds = time_in_queue / 1000.0; + + let disk_metrics = [ + ( + "node_disk_reads_completed_total", + rd_ios, + MetricType::Counter, + "{operations}", + "The total number of reads completed successfully", + ), + ( + "node_disk_reads_merged_total", + rd_merges, + MetricType::Counter, + "{operations}", + "The total number of reads merged", + ), + ( + "node_disk_read_bytes_total", + rd_bytes, + MetricType::Counter, + "By", + "The total number of bytes read successfully", + ), + ( + "node_disk_read_time_seconds_total", + rd_time_seconds, + MetricType::Counter, + "s", + "The total number of seconds spent by all reads", + ), + ( + "node_disk_writes_completed_total", + wr_ios, + MetricType::Counter, + "{operations}", + "The total number of writes completed successfully", + ), + ( + "node_disk_writes_merged_total", + wr_merges, + MetricType::Counter, + "{operations}", + "The total number of writes merged", + ), + ( + "node_disk_written_bytes_total", + wr_bytes, + MetricType::Counter, + "By", + "The total number of bytes written successfully", + ), + ( + "node_disk_write_time_seconds_total", + wr_time_seconds, + MetricType::Counter, + "s", + "The total number of seconds spent by all writes", + ), + ( + "node_disk_io_now", + ios_in_progress, + MetricType::Gauge, + "{operations}", + "The number of I/Os currently in progress", + ), + ( + "node_disk_io_time_seconds_total", + io_time_seconds, + MetricType::Counter, + "s", + "Total seconds spent doing I/Os", + ), + ( + "node_disk_io_time_weighted_seconds_total", + time_in_queue_seconds, + MetricType::Counter, + "s", + "The weighted number of seconds spent doing I/Os", + ), + ]; + + for (name, value, metric_type, unit, description) in disk_metrics { + if config.should_include_metric(name) { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type, + unit: Some(unit.to_string()), + description: Some(description.to_string()), + }); + } + } + + // Discard metrics (kernel 4.18+, fields 14-17) + if let Some((discard_ios, discard_merges, discard_sectors, discard_ticks)) = discard { + let discard_time_seconds = discard_ticks / 1000.0; + + let discard_metrics = [ + ( + "node_disk_discards_completed_total", + discard_ios, + "{operations}", + "The total number of discards completed successfully", + ), + ( + "node_disk_discards_merged_total", + discard_merges, + "{operations}", + "The total number of discards merged", + ), + ( + "node_disk_discarded_sectors_total", + discard_sectors, + "{sectors}", + "The total number of sectors discarded successfully", + ), + ( + "node_disk_discard_time_seconds_total", + discard_time_seconds, + "s", + "The total number of seconds spent discarding", + ), + ]; + + for (name, value, unit, description) in discard_metrics { + if config.should_include_metric(name) { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type: MetricType::Counter, + unit: Some(unit.to_string()), + description: Some(description.to_string()), + }); + } + } + } + + // Flush metrics (kernel 5.5+, fields 18-19) + if let Some((flush_ios, flush_ticks)) = flush { + let flush_time_seconds = flush_ticks / 1000.0; + + let flush_metrics = [ + ( + "node_disk_flush_requests_total", + flush_ios, + "{operations}", + "The total number of flush requests completed successfully", + ), + ( + "node_disk_flush_requests_time_seconds_total", + flush_time_seconds, + "s", + "The total number of seconds spent flushing", + ), + ]; + + for (name, value, unit, description) in flush_metrics { + if config.should_include_metric(name) { + metrics.push(CollectedMetric { + name: name.to_string(), + value, + labels: labels.clone(), + metric_type: MetricType::Counter, + unit: Some(unit.to_string()), + description: Some(description.to_string()), + }); + } + } + } + } + + metrics + } + + /// Collect virtual memory statistics from /proc/vmstat + pub(super) fn collect_vmstat(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/vmstat", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + for line in content.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 2 { + continue; + } + + let field_name = parts[0]; + let value: f64 = match parts[1].parse() { + Ok(v) => v, + Err(_) => continue, + }; + + // Convert field name to metric name: "pgfault" -> "node_vmstat_pgfault" + let metric_name = format!("node_vmstat_{}", field_name); + + if config.should_include_metric(&metric_name) { + // Most vmstat fields are counters, but nr_* fields are generally gauges + // (they represent current page counts, not cumulative events). + // The workingset_* family includes both gauges (workingset_nodes) + // and counters (workingset_refault_*). We treat nr_* as gauges except + // for the cumulative event counters that also carry the nr_ prefix; for + // workingset_* we whitelist the known gauge. + // numa_* fields (numa_hit, numa_miss, etc.) are cumulative counters. + let is_nr_counter = matches!( + field_name, + "nr_dirtied" + | "nr_written" + | "nr_throttled_written" + | "nr_vmscan_write" + | "nr_vmscan_immediate_reclaim" + | "nr_foll_pin_acquired" + | "nr_foll_pin_released" + ) || field_name.starts_with("nr_tlb_"); + + let metric_type = if !is_nr_counter + && (field_name.starts_with("nr_") || field_name == "workingset_nodes") + { + MetricType::Gauge + } else { + MetricType::Counter + }; + + metrics.push(CollectedMetric { + name: metric_name, + value, + labels: Vec::new(), + metric_type, + unit: None, + description: None, + }); + } + } + + metrics + } + + /// Collect network statistics from /proc/net/netstat and /proc/net/snmp. + /// + /// Both files share the same alternating header/value line format. + /// `/proc/net/netstat` contains TcpExt, IpExt, MPTcpExt counters. + /// `/proc/net/snmp` contains Tcp, Udp, Ip, Icmp, etc. — including the + /// commonly monitored gauge `CurrEstab`. + pub(super) fn collect_netstat(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + // Known gauge fields in /proc/net/snmp (all others are counters) + let snmp_gauges = [ + "CurrEstab", + "RtoAlgorithm", + "RtoMin", + "RtoMax", + "MaxConn", + "Forwarding", + "DefaultTTL", + ]; + + for subpath in ["net/netstat", "net/snmp"] { + let path = format!("{}/{}", self.procfs_path, subpath); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + continue; + } + }; + + // File format: alternating lines of "Protocol: key1 key2 ..." and "Protocol: val1 val2 ..." + let lines: Vec<&str> = content.lines().collect(); + let mut i = 0; + + while i + 1 < lines.len() { + let header_line = lines[i]; + let value_line = lines[i + 1]; + + let header_parts: Vec<&str> = header_line.split_whitespace().collect(); + let value_parts: Vec<&str> = value_line.split_whitespace().collect(); + + // Advance by one, not two, when the pair does not line up: an unexpected + // or missing line would otherwise desynchronise every remaining protocol + // in the file instead of resynchronising on the next header. + if header_parts.is_empty() || value_parts.is_empty() { + debug!("Skipping unpaired line {} in {}", i, path); + i += 1; + continue; + } + + // Preserve original casing to match Prometheus node_exporter + // conventions (e.g. "Tcp", "Udp", "TcpExt", "IpExt"). + let protocol = header_parts[0].trim_end_matches(':'); + + let value_protocol = value_parts[0].trim_end_matches(':'); + if protocol != value_protocol { + debug!( + "Skipping line {} in {}: expected values for {}, found {}", + i, path, protocol, value_protocol + ); + i += 1; + continue; + } + + for j in 1..header_parts.len().min(value_parts.len()) { + let key = header_parts[j]; + let value: f64 = match value_parts[j].parse() { + Ok(v) => v, + Err(_) => continue, + }; + + let metric_name = format!("node_netstat_{}_{}", protocol, key); + + if config.should_include_metric(&metric_name) { + let metric_type = if snmp_gauges.contains(&key) { + MetricType::Gauge + } else { + MetricType::Counter + }; + + metrics.push(CollectedMetric { + name: metric_name, + value, + labels: Vec::new(), + metric_type, + unit: None, + description: None, + }); + } + } + + i += 2; + } + } + + metrics + } + + /// Collect socket statistics from /proc/net/sockstat and /proc/net/sockstat6 + pub(super) fn collect_sockstat(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + let page_size = self.page_size; + + for subpath in ["net/sockstat", "net/sockstat6"] { + let path = format!("{}/{}", self.procfs_path, subpath); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + continue; + } + }; + + // Preserve the original casing to match Prometheus node_exporter + // conventions (e.g. "TCP", "UDP", "FRAG", "TCP6"). + for line in content.lines() { + let parts: Vec<&str> = line.split_whitespace().collect(); + if parts.len() < 3 { + continue; + } + + // Format: "protocol: key1 val1 key2 val2 ..." + let protocol = parts[0].trim_end_matches(':'); + + // Parse key-value pairs + let mut j = 1; + while j + 1 < parts.len() { + let key = parts[j]; + let raw_value: f64 = match parts[j + 1].parse() { + Ok(v) => v, + Err(_) => { + j += 2; + continue; + } + }; + + // Emit the raw metric + let metric_name = format!("node_sockstat_{}_{}", protocol, key); + if config.should_include_metric(&metric_name) { + metrics.push(CollectedMetric { + name: metric_name, + value: raw_value, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: None, + description: None, + }); + } + + // For "mem" fields, also emit a _bytes metric (pages -> bytes) + if key == "mem" { + let bytes_name = format!("node_sockstat_{}_{}_bytes", protocol, key); + if config.should_include_metric(&bytes_name) { + metrics.push(CollectedMetric { + name: bytes_name, + value: raw_value * page_size, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("By".to_string()), + description: None, + }); + } + } + + j += 2; + } + } + } + + metrics + } + + /// Collect file descriptor statistics from /proc/sys/fs/file-nr + pub(super) fn collect_filefd(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let path = format!("{}/sys/fs/file-nr", self.procfs_path); + let content = match fs::read_to_string(&path) { + Ok(c) => c, + Err(e) => { + debug!("Failed to read {}: {}", path, e); + return metrics; + } + }; + + // Format: "allocated_fds free_fds max_fds" + let parts: Vec<&str> = content.split_whitespace().collect(); + if parts.len() < 3 { + return metrics; + } + + // Skip a field that does not parse rather than reporting a misleading 0 + // (0 allocated descriptors is a plausible-looking but impossible value). + // parts[1] is "free" — always 0 on kernels ≥2.6, not exposed by node_exporter + if config.should_include_metric("node_filefd_allocated") + && let Ok(allocated) = parts[0].parse::() + { + metrics.push(CollectedMetric { + name: "node_filefd_allocated".to_string(), + value: allocated, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("{file_descriptors}".to_string()), + description: Some("Number of allocated file descriptors".to_string()), + }); + } + + if config.should_include_metric("node_filefd_maximum") + && let Ok(max) = parts[2].parse::() + { + metrics.push(CollectedMetric { + name: "node_filefd_maximum".to_string(), + value: max, + labels: Vec::new(), + metric_type: MetricType::Gauge, + unit: Some("{file_descriptors}".to_string()), + description: Some("Maximum number of file descriptors".to_string()), + }); + } + + metrics + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::receivers::node_metrics::collector::test_support::*; + use crate::receivers::node_metrics::config::Collector; + use tracing_test::traced_test; + + #[test] + fn test_collect_cpu() { + let tmp = tempfile::tempdir().unwrap(); + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let tck = collector.clk_tck; + + let metrics = collector.collect_cpu(&config_for(vec![Collector::Cpu]), Some(STAT_FIXTURE)); + + // The aggregate "cpu" line yields no data point: only per-core lines are + // reported, matching Prometheus node_exporter. + assert_eq!(label_values(&metrics, "cpu"), ["0", "1", "2"]); + + let user0 = find_one( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "0"), ("mode", "user")], + ); + assert_eq!(user0.value, 1000.0 / tck); + assert_eq!(user0.metric_type, MetricType::Counter); + assert_eq!(user0.unit.as_deref(), Some("s")); + + assert_eq!( + find_one( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "1"), ("mode", "idle")] + ) + .value, + 800_000.0 / tck + ); + assert_eq!( + find_one( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "0"), ("mode", "steal")] + ) + .value, + 8.0 / tck + ); + + // The guest fields go to their own metric, not to node_cpu_seconds_total. + assert_eq!( + find_one( + &metrics, + "node_cpu_guest_seconds_total", + &[("cpu", "0"), ("mode", "user")] + ) + .value, + 900.0 / tck + ); + assert_eq!( + find_one( + &metrics, + "node_cpu_guest_seconds_total", + &[("cpu", "0"), ("mode", "nice")] + ) + .value, + 10.0 / tck + ); + assert!( + matching( + &metrics, + "node_cpu_seconds_total", + &[("cpu", "0"), ("mode", "guest")] + ) + .is_empty() + ); + + // A truncated line reports exactly the fields it has: absent columns are + // skipped rather than reported as a counter-resetting zero. + let cpu2_modes: Vec<&str> = metrics + .iter() + .filter(|m| label(m, "cpu") == "2") + .map(|m| label(m, "mode")) + .collect(); + assert_eq!(cpu2_modes, ["user", "nice", "system"]); + assert!(matching(&metrics, "node_cpu_guest_seconds_total", &[("cpu", "2")]).is_empty()); + } + + #[test] + fn test_collect_meminfo() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "meminfo", + "\ +MemTotal: 16 kB +MemFree: 8192 kB +Active(anon): 4 kB +HugePages_Total: 0 +Broken: notanumber kB +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_meminfo(&config_for(vec![Collector::Memory])); + + let total = find_one(&metrics, "node_memory_MemTotal_bytes", &[]); + assert_eq!(total.value, 16384.0); + assert_eq!(total.unit.as_deref(), Some("By")); + assert_eq!(total.metric_type, MetricType::Gauge); + + // Parenthesised suffixes are normalised into the metric name. + assert_eq!( + find_one(&metrics, "node_memory_Active_anon_bytes", &[]).value, + 4096.0 + ); + + // A field without the "kB" unit keeps its raw value, gets no _bytes + // suffix and carries no OTLP unit. + let hugepages = find_one(&metrics, "node_memory_HugePages_Total", &[]); + assert_eq!(hugepages.value, 0.0); + assert_eq!(hugepages.unit, None); + + // The unparseable line is skipped entirely, leaving only the four good ones. + assert!( + !metrics.iter().any(|m| m.name.contains("Broken")), + "garbage line should be skipped, got {:?}", + metrics.iter().map(|m| &m.name).collect::>() + ); + assert_eq!(metrics.len(), 4); + } + + #[test] + fn test_collect_netdev() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "net/dev", + "\ +Inter-| Receive | Transmit + face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed + eth0: 123456789 98765 1 2 3 4 5 6 987654321 87654 7 8 9 10 11 12 + lo: 1000 10 0 0 0 0 0 0 1000 10 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_netdev(&config_for(vec![Collector::Network])); + + // Only eth0 is reported: the two header lines carry no counters and the + // 10-column "lo" line is too short to be trusted. + assert_eq!(label_values(&metrics, "device"), ["eth0"]); + assert_eq!(metrics.len(), 16); + + let rx_bytes = find_one( + &metrics, + "node_network_receive_bytes_total", + &[("device", "eth0")], + ); + assert_eq!(rx_bytes.value, 123_456_789.0); + assert_eq!(rx_bytes.unit.as_deref(), Some("By")); + assert_eq!(rx_bytes.metric_type, MetricType::Counter); + + // Transmit starts at column 9, not wherever the receive block happened to end. + assert_eq!( + find_one( + &metrics, + "node_network_transmit_bytes_total", + &[("device", "eth0")] + ) + .value, + 987_654_321.0 + ); + assert_eq!( + find_one( + &metrics, + "node_network_receive_multicast_total", + &[("device", "eth0")] + ) + .value, + 6.0 + ); + assert_eq!( + find_one( + &metrics, + "node_network_transmit_compressed_total", + &[("device", "eth0")] + ) + .value, + 12.0 + ); + } + + #[test] + fn test_collect_diskstats() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "diskstats", + "\ + 8 0 sda 100 10 2000 3000 200 20 4000 5000 0 6000 7000 + 8 1 sda1 50 5 1000 1500 100 10 2000 2500 0 3000 3500 + 8 16 sdb 100 10 abc 3000 200 20 4000 5000 0 6000 7000 + 7 0 loop0 1 0 8 2 0 0 0 0 0 4 2 + 252 0 zram0 5 0 40 1 6 0 48 2 0 3 3 + 253 0 vda 10 1 200 300 20 2 400 500 0 600 700 1 2 3 400 + 259 0 nvme0n1 1000 100 20000 30000 2000 200 40000 50000 0 60000 70000 5 6 7 800 9 1000 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_diskstats(&config_for(vec![Collector::Diskstats])); + + // Partitions (sda1), virtual devices (loop0, zram0) and the device with a + // corrupt core field (sdb) contribute nothing at all. + assert_eq!(label_values(&metrics, "device"), ["nvme0n1", "sda", "vda"]); + + let sda = [("device", "sda")]; + + // Sectors are 512 bytes; the tick fields are milliseconds. + assert_eq!( + find_one(&metrics, "node_disk_read_bytes_total", &sda).value, + 2000.0 * 512.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_written_bytes_total", &sda).value, + 4000.0 * 512.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_read_time_seconds_total", &sda).value, + 3.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_io_time_weighted_seconds_total", &sda).value, + 7.0 + ); + + // In-flight I/O is a point-in-time reading, unlike its siblings. + let io_now = find_one(&metrics, "node_disk_io_now", &sda); + assert_eq!(io_now.value, 0.0); + assert_eq!(io_now.metric_type, MetricType::Gauge); + assert_eq!( + find_one(&metrics, "node_disk_reads_completed_total", &sda).metric_type, + MetricType::Counter + ); + assert_eq!( + find_one(&metrics, "node_disk_io_time_seconds_total", &sda).metric_type, + MetricType::Counter + ); + + // 14 fields: no discard group and no flush group. + assert!(matching(&metrics, "node_disk_discards_completed_total", &sda).is_empty()); + assert!(matching(&metrics, "node_disk_flush_requests_total", &sda).is_empty()); + + // 18 fields: discard only. + let vda = [("device", "vda")]; + assert_eq!( + find_one(&metrics, "node_disk_discarded_sectors_total", &vda).value, + 3.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_discard_time_seconds_total", &vda).value, + 0.4 + ); + assert!(matching(&metrics, "node_disk_flush_requests_total", &vda).is_empty()); + + // 20 fields: discard and flush. + let nvme = [("device", "nvme0n1")]; + assert_eq!( + find_one(&metrics, "node_disk_discards_completed_total", &nvme).value, + 5.0 + ); + assert_eq!( + find_one(&metrics, "node_disk_flush_requests_total", &nvme).value, + 9.0 + ); + assert_eq!( + find_one( + &metrics, + "node_disk_flush_requests_time_seconds_total", + &nvme + ) + .value, + 1.0 + ); + } + + #[test] + fn test_collect_stat_and_processes() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "stat", STAT_FIXTURE); + write(tmp.path(), "sys/kernel/threads-max", "126988\n"); + write(tmp.path(), "sys/kernel/pid_max", "4194304\n"); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let config = config_for(vec![Collector::Stat, Collector::Processes]); + + let stat = std::fs::read_to_string(tmp.path().join("stat")).unwrap(); + let mut metrics = collector.collect_stat(&config, Some(&stat)); + metrics.extend(collector.collect_processes(&config)); + + // Only the first number on the "intr" line is the total; the per-vector + // counts that follow are not summed into it. + let intr = find_one(&metrics, "node_intr_total", &[]); + assert_eq!(intr.value, 1234.0); + assert_eq!(intr.metric_type, MetricType::Counter); + assert_eq!(intr.unit.as_deref(), Some("{interrupts}")); + + let forks = find_one(&metrics, "node_forks_total", &[]); + assert_eq!(forks.value, 4242.0); + assert_eq!(forks.metric_type, MetricType::Counter); + + let ctxt = find_one(&metrics, "node_context_switches_total", &[]); + assert_eq!(ctxt.value, 987_654_321.0); + assert_eq!(ctxt.metric_type, MetricType::Counter); + + // Process counts are instantaneous readings, not cumulative. + let running = find_one(&metrics, "node_procs_running", &[]); + assert_eq!(running.value, 3.0); + assert_eq!(running.metric_type, MetricType::Gauge); + + let blocked = find_one(&metrics, "node_procs_blocked", &[]); + assert_eq!(blocked.value, 1.0); + assert_eq!(blocked.metric_type, MetricType::Gauge); + + // The two limits come from /proc/sys/kernel, not from /proc/stat. + assert_eq!( + find_one(&metrics, "node_processes_max_threads", &[]).value, + 126_988.0 + ); + assert_eq!( + find_one(&metrics, "node_processes_max_processes", &[]).value, + 4_194_304.0 + ); + } + + #[test] + fn test_collect_sockstat() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "net/sockstat", + "\ +sockets: used 100 +TCP: inuse 1 orphan 0 tw 0 alloc 2 mem 5 +UDP: inuse 3 mem 1 +", + ); + write( + tmp.path(), + "net/sockstat6", + "\ +TCP6: inuse 2 +UDP6: inuse 1 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let page_size = collector.page_size; + let metrics = collector.collect_sockstat(&config_for(vec![Collector::Sockstat])); + + assert_eq!( + find_one(&metrics, "node_sockstat_sockets_used", &[]).value, + 100.0 + ); + assert_eq!( + find_one(&metrics, "node_sockstat_TCP_inuse", &[]).value, + 1.0 + ); + assert_eq!( + find_one(&metrics, "node_sockstat_TCP_alloc", &[]).value, + 2.0 + ); + + // "mem" is a page count; the derived _bytes metric scales it by the page size. + assert_eq!(find_one(&metrics, "node_sockstat_TCP_mem", &[]).value, 5.0); + let mem_bytes = find_one(&metrics, "node_sockstat_TCP_mem_bytes", &[]); + assert_eq!(mem_bytes.value, 5.0 * page_size); + assert_eq!(mem_bytes.unit.as_deref(), Some("By")); + assert_eq!(mem_bytes.metric_type, MetricType::Gauge); + + // The IPv6 file contributes its own protocol names. + assert_eq!( + find_one(&metrics, "node_sockstat_TCP6_inuse", &[]).value, + 2.0 + ); + assert_eq!( + find_one(&metrics, "node_sockstat_UDP6_inuse", &[]).value, + 1.0 + ); + assert!(matching(&metrics, "node_sockstat_TCP6_mem_bytes", &[]).is_empty()); + } + + #[test] + fn test_collect_filefd() { + let tmp = tempfile::tempdir().unwrap(); + write( + tmp.path(), + "sys/fs/file-nr", + "1234\t0\t9223372036854775807\n", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_filefd(&config_for(vec![Collector::Filefd])); + + let allocated = find_one(&metrics, "node_filefd_allocated", &[]); + assert_eq!(allocated.value, 1234.0); + assert_eq!(allocated.metric_type, MetricType::Gauge); + assert_eq!(allocated.unit.as_deref(), Some("{file_descriptors}")); + assert_eq!( + find_one(&metrics, "node_filefd_maximum", &[]).value, + 9_223_372_036_854_775_807f64 + ); + + // A field that does not parse is skipped, not reported as an impossible 0. + let broken = tempfile::tempdir().unwrap(); + write(broken.path(), "sys/fs/file-nr", "notanumber\t0\t100\n"); + let collector = SystemCollector::new(broken.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_filefd(&config_for(vec![Collector::Filefd])); + assert!(matching(&metrics, "node_filefd_allocated", &[]).is_empty()); + assert_eq!(find_one(&metrics, "node_filefd_maximum", &[]).value, 100.0); + } + + #[test] + fn test_collect_filesystem() { + let tmp = tempfile::tempdir().unwrap(); + let data = tmp.path().join("data"); + let excluded = tmp.path().join("excluded"); + let labelled = tmp.path().join("labelled"); + let merged = tmp.path().join("merged"); + let remote = tmp.path().join("remote"); + let ubi = tmp.path().join("ubi"); + let readonly_dir = tmp.path().join("readonly"); + for dir in [ + &data, + &excluded, + &labelled, + &merged, + &remote, + &ubi, + &readonly_dir, + ] { + std::fs::create_dir_all(dir).unwrap(); + } + let (data, excluded, labelled, merged, remote, ubi, readonly) = ( + data.to_str().unwrap(), + excluded.to_str().unwrap(), + labelled.to_str().unwrap(), + merged.to_str().unwrap(), + remote.to_str().unwrap(), + ubi.to_str().unwrap(), + readonly_dir.to_str().unwrap(), + ); + + // The mount points are real directories so the collector's statfs(2) call + // succeeds; only the fstype and device columns are synthetic. + let mounts = format!( + "/dev/sda2 {data} ext4 rw,relatime 0 0\n\ + /dev/sda2 {data} ext4 rw,relatime 0 0\n\ + proc {data} proc rw,nosuid,nodev,noexec,relatime 0 0\n\ + sysfs {data} sysfs rw,nosuid,nodev,noexec,relatime 0 0\n\ + fileserver:/exports/home {data} nfs4 rw,relatime 0 0\n\ + /dev/sdb1 {excluded} ext4 rw,relatime 0 0\n\ + /dev/disk/by-label/My\\040Disk {labelled} ext4 rw,relatime 0 0\n\ + mergerfs {merged} fuse.mergerfs rw,relatime 0 0\n\ + user@fileserver:/srv {remote} fuse.sshfs rw,relatime 0 0\n\ + ubi0:rootfs {ubi} ubifs rw,relatime 0 0\n\ + /dev/sdc1 {readonly} ext4 ro,relatime 0 0\n" + ); + write(tmp.path(), "mounts", &mounts); + + let config = NodeMetricsConfig { + collectors: vec![Collector::Filesystem], + filesystem_mount_exclude: Some( + regex::Regex::new(&format!("^{}$", regex::escape(excluded))).unwrap(), + ), + ..Default::default() + }; + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_filesystem(&config); + + // proc/sysfs are dropped by fstype, nfs4 and fuse.sshfs as remote filesystems + // whose statfs can block, the repeated (device, mountpoint) pair as a duplicate, + // and `excluded` by filesystem_mount_exclude. + assert_eq!( + label_values(&metrics, "fstype"), + ["ext4", "fuse.mergerfs", "ubifs"] + ); + // The survivors share the temp-dir prefix, so they sort by directory name. + assert_eq!( + label_values(&metrics, "mountpoint"), + [ + data.to_string(), + labelled.to_string(), + merged.to_string(), + readonly.to_string(), + ubi.to_string() + ] + ); + + let size = find_one( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", data)], + ); + assert_eq!(size.value, statfs_size_bytes(data)); + assert_eq!(size.unit.as_deref(), Some("By")); + assert_eq!(size.metric_type, MetricType::Gauge); + assert_eq!(label(size, "device"), "/dev/sda2"); + assert_eq!(label(size, "fstype"), "ext4"); + + // The remaining statfs-derived metrics are reported for the same mount. + for name in [ + "node_filesystem_free_bytes", + "node_filesystem_avail_bytes", + "node_filesystem_files", + "node_filesystem_files_free", + "node_filesystem_readonly", + ] { + find_one(&metrics, name, &[("mountpoint", data)]); + } + + // The read-only state comes from the mount options, since `statfs.f_flags` exists + // only on the BSDs and not on Linux. + assert_eq!( + find_one( + &metrics, + "node_filesystem_readonly", + &[("mountpoint", data)] + ) + .value, + 0.0 + ); + assert_eq!( + find_one( + &metrics, + "node_filesystem_readonly", + &[("mountpoint", readonly)] + ) + .value, + 1.0 + ); + + // Octal escapes in the device column are decoded before labelling. + assert_eq!( + label( + find_one( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", labelled)] + ), + "device" + ), + "/dev/disk/by-label/My Disk" + ); + + // FUSE mounts are named by subtype, and only the network-backed subtypes are + // excluded: a local FUSE filesystem is as worth reporting as any other, so the + // exclusion cannot be a blanket match on the "fuse" prefix. + let local_fuse = find_one( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", merged)], + ); + assert_eq!(label(local_fuse, "fstype"), "fuse.mergerfs"); + assert_eq!(label(local_fuse, "device"), "mergerfs"); + assert_eq!(local_fuse.value, statfs_size_bytes(merged)); + + // ...while the network-backed subtype is skipped, because statfs on a mount + // whose server has gone away blocks uninterruptibly on a shared blocking thread. + assert!( + matching( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", remote)] + ) + .is_empty(), + "a remote FUSE mount must not be scraped" + ); + + // ubifs is the on-device filesystem of the embedded targets this receiver runs + // on, so losing it would be the worst possible regression. + let flash = find_one( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", ubi)], + ); + assert_eq!(label(flash, "fstype"), "ubifs"); + assert_eq!(label(flash, "device"), "ubi0:rootfs"); + assert_eq!(flash.value, statfs_size_bytes(ubi)); + assert_eq!(flash.unit.as_deref(), Some("By")); + } + + /// The mount table comes from PID 1, not from `self`: with a bind-mounted host procfs + /// the `self` table is the *reader's* namespace, which would report a container's own + /// bind mounts labelled as if they were the host's filesystems. + #[test] + fn test_collect_filesystem_prefers_pid1_mount_table() { + let procfs = tempfile::tempdir().unwrap(); + let root = tempfile::tempdir().unwrap(); + let host_mount = root.path().join("data"); + std::fs::create_dir_all(&host_mount).unwrap(); + std::fs::create_dir_all(procfs.path().join("1")).unwrap(); + + // What PID 1 sees (the host) versus what this process sees (a container). + write( + &procfs.path().join("1"), + "mounts", + "/dev/sda1 /data ext4 rw 0 0\n", + ); + write( + procfs.path(), + "mounts", + "/dev/sdb1 /container-only ext4 rw 0 0\n", + ); + + let collector = SystemCollector::new(procfs.path().to_str().unwrap(), "/sys"); + let config = NodeMetricsConfig { + collectors: vec![Collector::Filesystem], + procfs_path: procfs.path().to_str().unwrap().to_string(), + // The host root is reachable under this prefix, so statfs is composed onto it + rootfs_path: root.path().to_str().unwrap().to_string(), + ..Default::default() + }; + + let metrics = collector.collect_filesystem(&config); + + // The host's mount is reported, labelled with the host's own path... + let size = find_one( + &metrics, + "node_filesystem_size_bytes", + &[ + ("device", "/dev/sda1"), + ("fstype", "ext4"), + ("mountpoint", "/data"), + ], + ); + assert!(size.value > 0.0); + // ...and the container's own table is not consulted. + assert!( + matching( + &metrics, + "node_filesystem_size_bytes", + &[("device", "/dev/sdb1")] + ) + .is_empty(), + "the self mount table must not be used when PID 1's is readable" + ); + } + + /// The two halves of the mount-point exclusion: the kernel pseudo-filesystem paths + /// are excluded including the mount point itself, while the container/runtime paths + /// are excluded only *below* it — a dedicated volume mounted exactly at + /// `/var/lib/docker` is a real filesystem, and the one most likely to fill up. + /// + /// Those paths are absolute, so a fixture cannot place a real directory at them and + /// statfs will usually fail. What the test can observe is whether the collector got + /// as far as calling statfs at all: an excluded mount point is skipped before that, + /// and so logs no statfs failure and produces no metric. + #[traced_test] + #[test] + fn test_collect_filesystem_mount_exclusion_boundaries() { + let tmp = tempfile::tempdir().unwrap(); + + // Every line uses a real, non-ignored fstype, so the only rule that can drop + // any of them is the mount-point exclusion under test. + write( + tmp.path(), + "mounts", + "\ +/dev/sda1 /dev ext4 rw,relatime 0 0 +/dev/sda2 /dev/shm ext4 rw,relatime 0 0 +/dev/sda3 /proc ext4 rw,relatime 0 0 +/dev/sda4 /sys ext4 rw,relatime 0 0 +/dev/sda5 /var/lib/docker ext4 rw,relatime 0 0 +/dev/sda6 /var/lib/docker/volumes/vol1 ext4 rw,relatime 0 0 +/dev/sda7 /var/lib/containers ext4 rw,relatime 0 0 +/dev/sda8 /var/lib/containers/storage ext4 rw,relatime 0 0 +/dev/sda9 /run/user/1000 ext4 rw,relatime 0 0 +/dev/sda10 /devices ext4 rw,relatime 0 0 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_filesystem(&config_for(vec![Collector::Filesystem])); + + // True when the mount point reached the statfs call: either it succeeded and + // produced a metric, or it failed and said so. + let reached_statfs = |mount_point: &str| { + !matching( + &metrics, + "node_filesystem_size_bytes", + &[("mountpoint", mount_point)], + ) + .is_empty() + || logs_contain(&format!("statfs({}) failed", mount_point)) + }; + + // Kernel pseudo-filesystems: the mount point itself is excluded, not just what + // is below it. `/dev` exists on every unix, so it would otherwise be reported. + for excluded in ["/dev", "/dev/shm", "/proc", "/sys"] { + assert!( + !reached_statfs(excluded), + "{} must be excluded before statfs is called", + excluded + ); + } + + // Container and runtime paths: excluded only below the mount point. + for excluded in [ + "/var/lib/docker/volumes/vol1", + "/var/lib/containers/storage", + "/run/user/1000", + ] { + assert!( + !reached_statfs(excluded), + "{} is below an excluded path and must be skipped", + excluded + ); + } + for reported in ["/var/lib/docker", "/var/lib/containers"] { + assert!( + reached_statfs(reported), + "a filesystem mounted exactly at {} is a real volume and must be scraped", + reported + ); + } + + // The exclusions are path-boundary aware: /devices is not below /dev. + assert!( + reached_statfs("/devices"), + "/devices must not be excluded by the /dev prefix" + ); + } + + /// `node_uname_info` is an annotation metric, so its label *values* are + /// host-dependent — but the key set, the value, the unit and the type are not. + #[test] + fn test_collect_uname() { + let tmp = tempfile::tempdir().unwrap(); + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_uname(&config_for(vec![Collector::Uname])); + + assert_eq!(metrics.len(), 1); + let info = &metrics[0]; + assert_eq!(info.name, "node_uname_info"); + // An info metric carries no measurement: the labels are the payload. + assert_eq!(info.value, 1.0); + assert_eq!(info.unit, None); + assert_eq!(info.metric_type, MetricType::Gauge); + + // Order matters as well as membership: these keys are the series identity, and + // `domainname` exists only in the Linux `utsname`. + let keys: Vec<&str> = info.labels.iter().map(|(k, _)| k.as_str()).collect(); + #[cfg(target_os = "linux")] + assert_eq!( + keys, + [ + "sysname", + "nodename", + "release", + "version", + "machine", + "domainname" + ] + ); + #[cfg(not(target_os = "linux"))] + assert_eq!( + keys, + ["sysname", "nodename", "release", "version", "machine"] + ); + + // uname(2) always fills these, so an empty value would mean the fixed-size C + // character array was decoded wrongly rather than that the host is unusual. + for key in ["sysname", "release", "machine"] { + assert!( + !label(info, key).is_empty(), + "{} should be populated from uname(2)", + key + ); + } + + // A filtered-out uname metric is dropped entirely, not emitted unlabelled. + let filtered = config_for(vec![Collector::Uname]) + .with_exclude_filter("^node_uname_info$") + .unwrap(); + assert!(collector.collect_uname(&filtered).is_empty()); + } + + #[test] + fn test_collect_loadavg() { + let tmp = tempfile::tempdir().unwrap(); + write(tmp.path(), "loadavg", "0.52 0.58 0.59 2/1234 5678\n"); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_loadavg(&NodeMetricsConfig::default()); + + assert_eq!(metrics.len(), 3); + assert_eq!(find_one(&metrics, "node_load1", &[]).value, 0.52); + assert_eq!(find_one(&metrics, "node_load5", &[]).value, 0.58); + assert_eq!(find_one(&metrics, "node_load15", &[]).value, 0.59); + for metric in &metrics { + assert_eq!(metric.metric_type, MetricType::Gauge); + assert!(metric.labels.is_empty()); + assert_eq!(metric.unit, None); + } + } + + // --------------------------------------------------------------------- + // Gauge/counter classification + // --------------------------------------------------------------------- + + #[test] + fn test_vmstat_gauge_counter_classification() { + let tmp = tempfile::tempdir().unwrap(); + let vmstat_content = "\ +nr_free_pages 123456 +nr_anon_pages 654 +nr_zone_inactive_anon 7890 +nr_dirtied 11 +nr_written 12 +nr_throttled_written 13 +nr_vmscan_write 13 +nr_vmscan_immediate_reclaim 14 +nr_foll_pin_acquired 15 +nr_foll_pin_released 16 +nr_tlb_remote_flush 17 +nr_tlb_local_flush_all 18 +pgfault 999999 +pgmajfault 1234 +pswpin 100 +pswpout 200 +workingset_nodes 500 +workingset_refault_anon 300 +numa_hit 42 +oom_kill 1 +"; + std::fs::write(tmp.path().join("vmstat"), vmstat_content).unwrap(); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let config = NodeMetricsConfig::default(); + let metrics = collector.collect_vmstat(&config); + + let find = |name: &str| metrics.iter().find(|m| m.name == name); + + // nr_* fields are gauges + assert_eq!( + find("node_vmstat_nr_free_pages").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_vmstat_nr_zone_inactive_anon") + .unwrap() + .metric_type, + MetricType::Gauge + ); + + // ... except for the cumulative event counters that share the nr_ prefix. + // These are the control cases for the exception list: without it they would + // silently be exported as gauges and lose their monotonic semantics. + for (name, value) in [ + ("node_vmstat_nr_dirtied", 11.0), + ("node_vmstat_nr_written", 12.0), + ("node_vmstat_nr_throttled_written", 13.0), + ("node_vmstat_nr_vmscan_write", 13.0), + ("node_vmstat_nr_vmscan_immediate_reclaim", 14.0), + ("node_vmstat_nr_foll_pin_acquired", 15.0), + ("node_vmstat_nr_foll_pin_released", 16.0), + // Every nr_tlb_* field is a cumulative flush count. + ("node_vmstat_nr_tlb_remote_flush", 17.0), + ("node_vmstat_nr_tlb_local_flush_all", 18.0), + ] { + let metric = find(name).unwrap_or_else(|| panic!("{} not collected", name)); + assert_eq!(metric.metric_type, MetricType::Counter, "{}", name); + assert_eq!(metric.value, value, "{}", name); + } + + // A neighbouring page-count gauge, to pin that the exception list is + // name-exact rather than a blanket "nr_ is a counter" rule. + let anon = find("node_vmstat_nr_anon_pages").unwrap(); + assert_eq!(anon.metric_type, MetricType::Gauge); + assert_eq!(anon.value, 654.0); + + // vmstat fields carry no unit and no labels, whichever type they get. + for metric in &metrics { + assert_eq!(metric.unit, None, "{}", metric.name); + assert!(metric.labels.is_empty(), "{}", metric.name); + } + + // workingset_nodes is the one gauge exception + let workingset_nodes = find("node_vmstat_workingset_nodes").unwrap(); + assert_eq!(workingset_nodes.metric_type, MetricType::Gauge); + assert_eq!(workingset_nodes.value, 500.0); + + // workingset_refault_* are counters + let refault = find("node_vmstat_workingset_refault_anon").unwrap(); + assert_eq!(refault.metric_type, MetricType::Counter); + assert_eq!(refault.value, 300.0); + + // pg*, psw*, numa_*, oom_kill are counters + assert_eq!( + find("node_vmstat_pgfault").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_pgmajfault").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_pswpin").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_pswpout").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_numa_hit").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_vmstat_oom_kill").unwrap().metric_type, + MetricType::Counter + ); + } + + #[test] + fn test_netstat_snmp_gauge_counter_classification() { + let tmp = tempfile::tempdir().unwrap(); + let net_dir = tmp.path().join("net"); + std::fs::create_dir(&net_dir).unwrap(); + + // /proc/net/snmp contains CurrEstab (gauge) and InSegs (counter) + let snmp_content = "\ +Tcp: RtoAlgorithm RtoMin RtoMax MaxConn ActiveOpens PassiveOpens AttemptFails EstabResets CurrEstab InSegs OutSegs RetransSegs InErrs OutRsts InCsumErrors +Tcp: 1 200 120000 -1 500 100 10 5 42 99999 88888 100 2 50 0 +Ip: Forwarding DefaultTTL InReceives InHdrErrors +Ip: 1 64 123456 0 +"; + std::fs::write(net_dir.join("snmp"), snmp_content).unwrap(); + + // /proc/net/netstat — all fields are counters + let netstat_content = "\ +TcpExt: SyncookiesSent SyncookiesRecv SyncookiesFailed +TcpExt: 10 20 5 +"; + std::fs::write(net_dir.join("netstat"), netstat_content).unwrap(); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let config = NodeMetricsConfig::default(); + let metrics = collector.collect_netstat(&config); + + let find = |name: &str| metrics.iter().find(|m| m.name == name); + + // Known gauges from /proc/net/snmp + assert_eq!( + find("node_netstat_Tcp_CurrEstab").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Tcp_RtoAlgorithm").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Tcp_RtoMin").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Tcp_RtoMax").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Tcp_MaxConn").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Ip_Forwarding").unwrap().metric_type, + MetricType::Gauge + ); + assert_eq!( + find("node_netstat_Ip_DefaultTTL").unwrap().metric_type, + MetricType::Gauge + ); + + // Counters from /proc/net/snmp + assert_eq!( + find("node_netstat_Tcp_ActiveOpens").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_netstat_Tcp_InSegs").unwrap().metric_type, + MetricType::Counter + ); + assert_eq!( + find("node_netstat_Ip_InReceives").unwrap().metric_type, + MetricType::Counter + ); + + // Counters from /proc/net/netstat + assert_eq!( + find("node_netstat_TcpExt_SyncookiesSent") + .unwrap() + .metric_type, + MetricType::Counter + ); + } + + #[test] + fn test_netstat_resynchronises_after_an_unpaired_line() { + let tmp = tempfile::tempdir().unwrap(); + + // The IpExt header has no value line: the following line is the next + // protocol's header. Advancing by two would consume MPTcpExt's header as if + // it were a value line and lose every protocol after the anomaly. + write( + tmp.path(), + "net/netstat", + "\ +TcpExt: SyncookiesSent SyncookiesRecv +TcpExt: 10 20 +IpExt: InNoRoutes InTruncatedPkts +MPTcpExt: MPCapableSYNRX MPCapableACKRX +MPTcpExt: 7 8 +", + ); + + // Another shape of anomaly: a stray line the parser cannot pair with + // anything, sitting between two well-formed protocol blocks. A blank line + // follows, as a kernel with no MPTcpExt support leaves behind. + write( + tmp.path(), + "net/snmp", + "\ +Ip: Forwarding DefaultTTL InReceives +Ip: 1 64 123456 +some unexpected line the kernel never wrote +Udp: InDatagrams NoPorts +Udp: 100 5 + +Tcp: CurrEstab InSegs +Tcp: 42 99999 +", + ); + + let collector = SystemCollector::new(tmp.path().to_str().unwrap(), "/sys"); + let metrics = collector.collect_netstat(&NodeMetricsConfig::default()); + + // Everything before the anomaly is unaffected. + assert_eq!( + find_one(&metrics, "node_netstat_TcpExt_SyncookiesSent", &[]).value, + 10.0 + ); + assert_eq!( + find_one(&metrics, "node_netstat_TcpExt_SyncookiesRecv", &[]).value, + 20.0 + ); + + // The unpaired header contributes nothing, rather than pairing up with the + // next protocol's header and mislabelling its keys. + assert!(matching(&metrics, "node_netstat_IpExt_InNoRoutes", &[]).is_empty()); + assert!(matching(&metrics, "node_netstat_IpExt_InTruncatedPkts", &[]).is_empty()); + + // The protocol *after* the anomaly is still collected, with the right value: + // the parser resynchronised on the next header instead of staying one line + // out of step for the rest of the file. + let mptcp = find_one(&metrics, "node_netstat_MPTcpExt_MPCapableSYNRX", &[]); + assert_eq!(mptcp.value, 7.0); + assert_eq!(mptcp.metric_type, MetricType::Counter); + assert!(mptcp.labels.is_empty()); + assert_eq!(mptcp.unit, None); + assert_eq!( + find_one(&metrics, "node_netstat_MPTcpExt_MPCapableACKRX", &[]).value, + 8.0 + ); + + // Same in /proc/net/snmp: the block after the stray line survives, and the + // gauge/counter classification is still applied to the recovered keys. + assert_eq!( + find_one(&metrics, "node_netstat_Ip_InReceives", &[]).value, + 123_456.0 + ); + let udp = find_one(&metrics, "node_netstat_Udp_InDatagrams", &[]); + assert_eq!(udp.value, 100.0); + assert_eq!(udp.metric_type, MetricType::Counter); + assert_eq!( + find_one(&metrics, "node_netstat_Udp_NoPorts", &[]).value, + 5.0 + ); + + let curr_estab = find_one(&metrics, "node_netstat_Tcp_CurrEstab", &[]); + assert_eq!(curr_estab.value, 42.0); + assert_eq!(curr_estab.metric_type, MetricType::Gauge); + assert_eq!( + find_one(&metrics, "node_netstat_Tcp_InSegs", &[]).value, + 99_999.0 + ); + } +} diff --git a/src/receivers/node_metrics/collector/sysfs.rs b/src/receivers/node_metrics/collector/sysfs.rs new file mode 100644 index 00000000..dea86aea --- /dev/null +++ b/src/receivers/node_metrics/collector/sysfs.rs @@ -0,0 +1,991 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! sysfs-backed collectors: CPU frequency, thermal zones, NVMe and hwmon sensors. +//! +//! Metric names, HELP strings and the hwmon chip-naming approach are derived from +//! Prometheus node_exporter — see the notice in this module's `mod.rs`. + +use crate::receivers::node_metrics::collector::{CollectedMetric, MetricType, SystemCollector}; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use std::fs; +use tracing::debug; + +impl SystemCollector { + /// Collect CPU frequency metrics from /sys/devices/system/cpu/cpu*/cpufreq/ + pub(super) fn collect_cpufreq(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + let cpu_dir = format!("{}/devices/system/cpu", self.sysfs_path); + + let entries = match fs::read_dir(&cpu_dir) { + Ok(e) => e, + Err(e) => { + debug!("Failed to read {}: {}", cpu_dir, e); + return metrics; + } + }; + + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + // Match cpu0, cpu1, etc. + let cpu_num = match name_str.strip_prefix("cpu") { + Some(n) if n.chars().all(|c| c.is_ascii_digit()) && !n.is_empty() => n.to_string(), + _ => continue, + }; + + let cpufreq_dir = entry.path().join("cpufreq"); + + let freq_files = [ + ( + "scaling_cur_freq", + "node_cpu_scaling_frequency_hertz", + "Current scaled CPU frequency", + ), + ( + "scaling_min_freq", + "node_cpu_scaling_frequency_min_hertz", + "Minimum scaled CPU frequency", + ), + ( + "scaling_max_freq", + "node_cpu_scaling_frequency_max_hertz", + "Maximum scaled CPU frequency", + ), + ( + "cpuinfo_cur_freq", + "node_cpu_frequency_hertz", + "Current CPU frequency from hardware", + ), + ( + "cpuinfo_min_freq", + "node_cpu_frequency_min_hertz", + "Minimum CPU frequency from hardware", + ), + ( + "cpuinfo_max_freq", + "node_cpu_frequency_max_hertz", + "Maximum CPU frequency from hardware", + ), + ]; + + for (file, metric_name, description) in &freq_files { + if !config.should_include_metric(metric_name) { + continue; + } + + let path = cpufreq_dir.join(file); + if let Ok(content) = fs::read_to_string(&path) { + // Values in cpufreq are in kHz, convert to Hz + if let Ok(khz) = content.trim().parse::() { + metrics.push(CollectedMetric { + name: metric_name.to_string(), + value: khz * 1000.0, + labels: vec![("cpu".to_string(), cpu_num.clone())], + metric_type: MetricType::Gauge, + unit: Some("Hz".to_string()), + description: Some(description.to_string()), + }); + } + } + } + } + + metrics + } + + /// Collect thermal zone temperatures from /sys/class/thermal/thermal_zone*/ + pub(super) fn collect_thermal_zone(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + let thermal_dir = format!("{}/class/thermal", self.sysfs_path); + + let entries = match fs::read_dir(&thermal_dir) { + Ok(e) => e, + Err(e) => { + debug!("Failed to read {}: {}", thermal_dir, e); + return metrics; + } + }; + + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + let zone_num = match name_str.strip_prefix("thermal_zone") { + Some(n) if n.chars().all(|c| c.is_ascii_digit()) && !n.is_empty() => n.to_string(), + _ => continue, + }; + + let zone_path = entry.path(); + + // Read zone type (e.g., "x86_pkg_temp", "acpitz") + let zone_type = fs::read_to_string(zone_path.join("type")) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + + // Read temperature (millidegrees Celsius -> Celsius) + if config.should_include_metric("node_thermal_zone_temp") + && let Ok(content) = fs::read_to_string(zone_path.join("temp")) + && let Ok(millideg) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_thermal_zone_temp".to_string(), + value: millideg / 1000.0, + labels: vec![ + ("zone".to_string(), zone_num.clone()), + ("type".to_string(), zone_type.clone()), + ], + metric_type: MetricType::Gauge, + unit: Some("Cel".to_string()), + description: Some("Zone temperature in Celsius".to_string()), + }); + } + } + + // Cooling devices + if (config.should_include_metric("node_cooling_device_cur_state") + || config.should_include_metric("node_cooling_device_max_state")) + && let Ok(entries) = fs::read_dir(&thermal_dir) + { + for entry in entries.flatten() { + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + + // Require the trailing index, like the cpu/thermal_zone/nvme scans do + let is_cooling_device = name_str + .strip_prefix("cooling_device") + .is_some_and(|r| !r.is_empty() && r.chars().all(|c| c.is_ascii_digit())); + if !is_cooling_device { + continue; + } + + let dev_path = entry.path(); + // Use bare index ("0", "1") matching Prometheus node_exporter convention. + // The guard above already required the prefix, so this cannot be empty. + let dev_name = name_str + .strip_prefix("cooling_device") + .unwrap_or_default() + .to_string(); + + let dev_type = fs::read_to_string(dev_path.join("type")) + .map(|s| s.trim().to_string()) + .unwrap_or_default(); + + let labels = vec![ + ("name".to_string(), dev_name), + ("type".to_string(), dev_type), + ]; + + for (file, metric_name, desc) in [ + ( + "cur_state", + "node_cooling_device_cur_state", + "Current state of the cooling device", + ), + ( + "max_state", + "node_cooling_device_max_state", + "Maximum state of the cooling device", + ), + ] { + if !config.should_include_metric(metric_name) { + continue; + } + if let Ok(content) = fs::read_to_string(dev_path.join(file)) + && let Ok(val) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: metric_name.to_string(), + value: val, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: None, + description: Some(desc.to_string()), + }); + } + } + } + } + + metrics + } + + /// Collect NVMe device info from /sys/class/nvme/ + pub(super) fn collect_nvme(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + if !config.should_include_metric("node_nvme_info") { + return metrics; + } + + let nvme_dir = format!("{}/class/nvme", self.sysfs_path); + let entries = match fs::read_dir(&nvme_dir) { + Ok(e) => e, + Err(e) => { + debug!("Failed to read {}: {}", nvme_dir, e); + return metrics; + } + }; + + for entry in entries.flatten() { + let dev_path = entry.path(); + let dev_name = entry.file_name().to_string_lossy().to_string(); + + // Only real controller entries (nvme0, nvme1, ...), consistent with the + // cpufreq and thermal-zone collectors. A stray file in this directory would + // otherwise produce an info metric with five empty labels. + let is_controller = dev_name + .strip_prefix("nvme") + .is_some_and(|rest| !rest.is_empty() && rest.chars().all(|c| c.is_ascii_digit())); + if !is_controller { + debug!("Skipping non-controller entry {} in {}", dev_name, nvme_dir); + continue; + } + + let read_attr = |attr: &str| -> String { + fs::read_to_string(dev_path.join(attr)) + .map(|s| s.trim().to_string()) + .unwrap_or_default() + }; + + metrics.push(CollectedMetric { + name: "node_nvme_info".to_string(), + value: 1.0, + labels: vec![ + ("device".to_string(), dev_name), + ("firmware_revision".to_string(), read_attr("firmware_rev")), + ("model".to_string(), read_attr("model")), + ("serial".to_string(), read_attr("serial")), + ("state".to_string(), read_attr("state")), + ], + metric_type: MetricType::Gauge, + unit: None, + description: Some("NVMe device information".to_string()), + }); + } + + metrics + } + + /// Collect hardware monitoring sensor data from /sys/class/hwmon/ + pub(super) fn collect_hwmon(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + let hwmon_dir = format!("{}/class/hwmon", self.sysfs_path); + + let entries = match fs::read_dir(&hwmon_dir) { + Ok(e) => e, + Err(e) => { + debug!("Failed to read {}: {}", hwmon_dir, e); + return metrics; + } + }; + + for entry in entries.flatten() { + let hwmon_path = entry.path(); + let hwmon_dir_name = entry.file_name().to_string_lossy().to_string(); + + // Require the trailing index, like the cpu/thermal_zone/nvme/cooling_device + // scans do, so a stray file in this directory is not treated as a chip. + let is_hwmon_dir = hwmon_dir_name + .strip_prefix("hwmon") + .is_some_and(|rest| !rest.is_empty() && rest.chars().all(|c| c.is_ascii_digit())); + if !is_hwmon_dir { + debug!( + "Skipping non-chip entry {} in {}", + hwmon_dir_name, hwmon_dir + ); + continue; + } + + // The human-readable model name, published separately as an annotation metric. + // It is not usable as an identity on its own: it is a chip *model* and repeats + // across instances (two NVMe drives both report "nvme"). + let chip_name = fs::read_to_string(hwmon_path.join("name")) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + + // The `chip` label must uniquely identify the hwmon instance, otherwise two + // chips of the same model produce identical attribute sets and one of them is + // dropped. The backing device name is both unique and stable, so prefer it. + // Without a `device` symlink, fall back to an identity that still includes the + // hwmonN directory, which is unique within a host — the model name alone is + // not. That index depends on module load order, so a fallback identity is not + // stable across reboots. + let chip = fs::read_link(hwmon_path.join("device")) + .ok() + .and_then(|target| { + target + .file_name() + .map(|n| sanitize_hwmon_chip(&n.to_string_lossy())) + }) + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| match &chip_name { + Some(name) => format!("{}_{}", sanitize_hwmon_chip(name), hwmon_dir_name), + None => hwmon_dir_name.clone(), + }); + + if let Some(chip_name) = &chip_name + && config.should_include_metric("node_hwmon_chip_names") + { + metrics.push(CollectedMetric { + name: "node_hwmon_chip_names".to_string(), + value: 1.0, + labels: vec![ + ("chip".to_string(), chip.clone()), + ("chip_name".to_string(), chip_name.clone()), + ], + metric_type: MetricType::Gauge, + unit: None, + description: Some( + "Annotation metric for human-readable chip names".to_string(), + ), + }); + } + + // Scan for sensor files: temp*, in*, fan*, power* and curr* `_input` readings + let dir_entries = match fs::read_dir(&hwmon_path) { + Ok(e) => e, + Err(_) => continue, + }; + + for sensor_entry in dir_entries.flatten() { + let fname = sensor_entry.file_name(); + let fname_str = fname.to_string_lossy(); + + // Only process *_input files — these are the primary readings + let Some(sensor_name) = fname_str.strip_suffix("_input") else { + continue; + }; + + // hwmon sensor files are named _input. Match on the alphabetic + // prefix exactly rather than with `starts_with`, so that e.g. `intrusion0` + // is not mistaken for a voltage input (`in`). + let kind: &str = &sensor_name[..sensor_name + .find(|c: char| !c.is_ascii_alphabetic()) + .unwrap_or(sensor_name.len())]; + + let (metric_name, divisor, unit, desc) = match kind { + // temp1_input, temp2_input, etc. — millidegrees C + "temp" => ( + "node_hwmon_temp_celsius", + 1000.0, + "Cel", + "Hardware monitor temperature", + ), + // in0_input, in1_input, etc. — millivolts + "in" => ( + "node_hwmon_in_volts", + 1000.0, + "V", + "Hardware monitor voltage", + ), + // fan1_input, fan2_input, etc. — RPM (no conversion) + "fan" => ( + "node_hwmon_fan_rpm", + 1.0, + "{rev}/min", + "Hardware monitor fan speed", + ), + // power1_input, etc. — microwatts + "power" => ( + "node_hwmon_power_watts", + 1_000_000.0, + "W", + "Hardware monitor power consumption", + ), + // curr1_input, etc. — milliamps + "curr" => ( + "node_hwmon_curr_amps", + 1000.0, + "A", + "Hardware monitor current", + ), + _ => { + debug!( + "Skipping unsupported hwmon sensor {} on chip {}", + sensor_name, chip + ); + continue; + } + }; + + // The threshold metrics below check their own names, so this sensor is only + // skipped entirely when none of its metrics are wanted. Gating everything + // on `metric_name` would make the thresholds unreachable whenever the + // reading itself is filtered out. + let want_reading = config.should_include_metric(metric_name); + let want_thresholds = kind == "temp" + && (config.should_include_metric("node_hwmon_temp_max_celsius") + || config.should_include_metric("node_hwmon_temp_crit_celsius")); + if !want_reading && !want_thresholds { + continue; + } + + // The `sensor` label is the raw sysfs name (`temp1`, `in0`), which is unique + // within a chip, matching Prometheus node_exporter. The human-readable text + // from `_label` is published separately as an annotation metric: + // using it as the identity would collide whenever two sensors on one chip + // carry the same text, and would move series whenever a driver gains or + // changes a label file. + let labels = vec![ + ("chip".to_string(), chip.clone()), + ("sensor".to_string(), sensor_name.to_string()), + ]; + + let label_path = hwmon_path.join(format!("{}_label", sensor_name)); + if let Some(sensor_label) = fs::read_to_string(&label_path) + .ok() + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + && config.should_include_metric("node_hwmon_sensor_label") + { + metrics.push(CollectedMetric { + name: "node_hwmon_sensor_label".to_string(), + value: 1.0, + labels: vec![ + ("chip".to_string(), chip.clone()), + ("sensor".to_string(), sensor_name.to_string()), + ("label".to_string(), sensor_label), + ], + metric_type: MetricType::Gauge, + unit: None, + description: Some( + "Annotation metric for human-readable sensor labels".to_string(), + ), + }); + } + + if want_reading + && let Ok(content) = fs::read_to_string(sensor_entry.path()) + && let Ok(raw) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: metric_name.to_string(), + value: raw / divisor, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: Some(unit.to_string()), + description: Some(desc.to_string()), + }); + } + + // For temperature sensors, also read max and crit thresholds + if kind == "temp" { + let max_path = hwmon_path.join(format!("{}_max", sensor_name)); + if config.should_include_metric("node_hwmon_temp_max_celsius") + && let Ok(content) = fs::read_to_string(&max_path) + && let Ok(raw) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_hwmon_temp_max_celsius".to_string(), + value: raw / 1000.0, + labels: labels.clone(), + metric_type: MetricType::Gauge, + unit: Some("Cel".to_string()), + description: Some( + "Hardware monitor temperature max threshold".to_string(), + ), + }); + } + + let crit_path = hwmon_path.join(format!("{}_crit", sensor_name)); + if config.should_include_metric("node_hwmon_temp_crit_celsius") + && let Ok(content) = fs::read_to_string(&crit_path) + && let Ok(raw) = content.trim().parse::() + { + metrics.push(CollectedMetric { + name: "node_hwmon_temp_crit_celsius".to_string(), + value: raw / 1000.0, + labels, + metric_type: MetricType::Gauge, + unit: Some("Cel".to_string()), + description: Some( + "Hardware monitor temperature critical threshold".to_string(), + ), + }); + } + } + } + } + + metrics + } +} + +/// Make an hwmon chip identity safe to use as a metric label value. +/// +/// Non-alphanumeric characters are replaced with `_`, following the same approach as +/// Prometheus `node_exporter`. This is a lossy mapping (`coretemp.0` and `coretemp-0` +/// both become `coretemp_0`), so it does not by itself guarantee uniqueness — a backing +/// device name is unique on its own, and the fallback identities compose the hwmon +/// directory to stay so. Values also differ from node_exporter's, which prefixes the +/// device's parent directory (`platform_coretemp_0` where this produces `coretemp_0`). +fn sanitize_hwmon_chip(device_name: &str) -> String { + device_name + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::receivers::node_metrics::collector::test_support::*; + use crate::receivers::node_metrics::config::Collector; + use std::os::unix::fs::symlink; + + #[test] + fn test_collect_cpufreq() { + let sys = tempfile::tempdir().unwrap(); + write( + sys.path(), + "devices/system/cpu/cpu0/cpufreq/scaling_cur_freq", + "2400000\n", + ); + write( + sys.path(), + "devices/system/cpu/cpu0/cpufreq/scaling_min_freq", + "800000\n", + ); + write( + sys.path(), + "devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq", + "3600000\n", + ); + write( + sys.path(), + "devices/system/cpu/cpu1/cpufreq/scaling_cur_freq", + "1200000\n", + ); + // Siblings of the cpu directories that must not be scanned as CPUs. + write(sys.path(), "devices/system/cpu/cpufreq/boost", "1\n"); + write( + sys.path(), + "devices/system/cpu/cpuidle/current_driver", + "none\n", + ); + write(sys.path(), "devices/system/cpu/cpu_capacity", "1024\n"); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_cpufreq(&config_for(vec![Collector::Cpufreq])); + + assert_eq!(label_values(&metrics, "cpu"), ["0", "1"]); + assert_eq!(metrics.len(), 4); + + // cpufreq reports kHz; the metric is Hz. + let cur = find_one( + &metrics, + "node_cpu_scaling_frequency_hertz", + &[("cpu", "0")], + ); + assert_eq!(cur.value, 2_400_000_000.0); + assert_eq!(cur.unit.as_deref(), Some("Hz")); + assert_eq!(cur.metric_type, MetricType::Gauge); + + assert_eq!( + find_one( + &metrics, + "node_cpu_scaling_frequency_min_hertz", + &[("cpu", "0")] + ) + .value, + 800_000_000.0 + ); + assert_eq!( + find_one(&metrics, "node_cpu_frequency_max_hertz", &[("cpu", "0")]).value, + 3_600_000_000.0 + ); + assert_eq!( + find_one( + &metrics, + "node_cpu_scaling_frequency_hertz", + &[("cpu", "1")] + ) + .value, + 1_200_000_000.0 + ); + } + + #[test] + fn test_collect_thermal_zone() { + let sys = tempfile::tempdir().unwrap(); + write( + sys.path(), + "class/thermal/thermal_zone0/type", + "x86_pkg_temp\n", + ); + write(sys.path(), "class/thermal/thermal_zone0/temp", "45000\n"); + write( + sys.path(), + "class/thermal/cooling_device0/type", + "Processor\n", + ); + write(sys.path(), "class/thermal/cooling_device0/cur_state", "0\n"); + write(sys.path(), "class/thermal/cooling_device0/max_state", "3\n"); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_thermal_zone(&config_for(vec![Collector::ThermalZone])); + + // Millidegrees Celsius in the file, degrees Celsius in the metric. + let temp = find_one( + &metrics, + "node_thermal_zone_temp", + &[("zone", "0"), ("type", "x86_pkg_temp")], + ); + assert_eq!(temp.value, 45.0); + assert_eq!(temp.unit.as_deref(), Some("Cel")); + assert_eq!(temp.metric_type, MetricType::Gauge); + + let cooling = [("name", "0"), ("type", "Processor")]; + assert_eq!( + find_one(&metrics, "node_cooling_device_cur_state", &cooling).value, + 0.0 + ); + assert_eq!( + find_one(&metrics, "node_cooling_device_max_state", &cooling).value, + 3.0 + ); + assert_eq!(metrics.len(), 3); + } + + #[test] + fn test_collect_nvme() { + let sys = tempfile::tempdir().unwrap(); + write( + sys.path(), + "class/nvme/nvme0/model", + "Samsung SSD 980 PRO 1TB\n", + ); + write(sys.path(), "class/nvme/nvme0/serial", "S5GXNX0R123456\n"); + write(sys.path(), "class/nvme/nvme0/firmware_rev", "5B2QGXA7\n"); + write(sys.path(), "class/nvme/nvme0/state", "live\n"); + // Entries that are not controllers: a namespace directory, and a plain file. + // Both carry none of the controller attributes, so accepting them would emit + // an info metric whose five labels are all empty. + write(sys.path(), "class/nvme/nvme0n1/size", "1953525168\n"); + write(sys.path(), "class/nvme/uevent", "\n"); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_nvme(&config_for(vec![Collector::Nvme])); + + assert_eq!(metrics.len(), 1); + assert_eq!(label_values(&metrics, "device"), ["nvme0"]); + let info = &metrics[0]; + assert_eq!(info.name, "node_nvme_info"); + assert_eq!(info.value, 1.0); + assert_eq!(info.metric_type, MetricType::Gauge); + assert_eq!(info.unit, None); + assert_eq!(label(info, "device"), "nvme0"); + assert_eq!(label(info, "model"), "Samsung SSD 980 PRO 1TB"); + assert_eq!(label(info, "serial"), "S5GXNX0R123456"); + assert_eq!(label(info, "firmware_revision"), "5B2QGXA7"); + assert_eq!(label(info, "state"), "live"); + + // The label set is exactly these five keys, so a stray entry cannot hide as + // an extra series with the same name. + let mut keys: Vec<&str> = info.labels.iter().map(|(k, _)| k.as_str()).collect(); + keys.sort_unstable(); + assert_eq!( + keys, + ["device", "firmware_revision", "model", "serial", "state"] + ); + } + + #[test] + fn test_collect_hwmon() { + let sys = tempfile::tempdir().unwrap(); + write(sys.path(), "class/hwmon/hwmon0/name", "coretemp\n"); + write(sys.path(), "class/hwmon/hwmon0/temp1_input", "42000\n"); + write( + sys.path(), + "class/hwmon/hwmon0/temp1_label", + "Package id 0\n", + ); + write(sys.path(), "class/hwmon/hwmon0/temp1_max", "84000\n"); + write(sys.path(), "class/hwmon/hwmon0/temp1_crit", "100000\n"); + // No label file at all. + write(sys.path(), "class/hwmon/hwmon0/temp2_input", "41000\n"); + // Present but empty label file. + write(sys.path(), "class/hwmon/hwmon0/temp3_input", "40000\n"); + write(sys.path(), "class/hwmon/hwmon0/temp3_label", "\n"); + write(sys.path(), "class/hwmon/hwmon0/in0_input", "1200\n"); + write(sys.path(), "class/hwmon/hwmon0/fan1_input", "1500\n"); + write(sys.path(), "class/hwmon/hwmon0/power1_input", "7500000\n"); + write(sys.path(), "class/hwmon/hwmon0/curr1_input", "850\n"); + // Not a voltage input, despite sharing the "in" prefix. + write(sys.path(), "class/hwmon/hwmon0/intrusion0_input", "0\n"); + symlink( + "../../devices/platform/coretemp.0", + sys.path().join("class/hwmon/hwmon0/device"), + ) + .unwrap(); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_hwmon(&config_for(vec![Collector::Hwmon])); + + // The chip identity comes from the backing device, sanitised for use as a label. + let chip = "coretemp_0"; + assert_eq!(label_values(&metrics, "chip"), [chip]); + + let names = find_one( + &metrics, + "node_hwmon_chip_names", + &[("chip", chip), ("chip_name", "coretemp")], + ); + assert_eq!(names.value, 1.0); + // Info-style annotation metrics carry no unit, like node_nvme_info/node_uname_info + assert_eq!(names.unit, None); + + // Temperatures, voltages and currents are milli-units; power is micro; + // fan speed is already RPM. + let temp1 = find_one( + &metrics, + "node_hwmon_temp_celsius", + &[("chip", chip), ("sensor", "temp1")], + ); + assert_eq!(temp1.value, 42.0); + assert_eq!(temp1.unit.as_deref(), Some("Cel")); + assert_eq!(temp1.metric_type, MetricType::Gauge); + + assert_eq!( + find_one( + &metrics, + "node_hwmon_temp_max_celsius", + &[("chip", chip), ("sensor", "temp1")] + ) + .value, + 84.0 + ); + assert_eq!( + find_one( + &metrics, + "node_hwmon_temp_crit_celsius", + &[("chip", chip), ("sensor", "temp1")] + ) + .value, + 100.0 + ); + + let volts = find_one(&metrics, "node_hwmon_in_volts", &[("sensor", "in0")]); + assert_eq!(volts.value, 1.2); + assert_eq!(volts.unit.as_deref(), Some("V")); + + let fan = find_one(&metrics, "node_hwmon_fan_rpm", &[("sensor", "fan1")]); + assert_eq!(fan.value, 1500.0); + assert_eq!(fan.unit.as_deref(), Some("{rev}/min")); + + let power = find_one(&metrics, "node_hwmon_power_watts", &[("sensor", "power1")]); + assert_eq!(power.value, 7.5); + assert_eq!(power.unit.as_deref(), Some("W")); + + let curr = find_one(&metrics, "node_hwmon_curr_amps", &[("sensor", "curr1")]); + assert_eq!(curr.value, 0.85); + assert_eq!(curr.unit.as_deref(), Some("A")); + + // Sensors are identified by their raw sysfs name whether or not they carry a label + // file, so two sensors on one chip can never collide. + assert_eq!( + find_one(&metrics, "node_hwmon_temp_celsius", &[("sensor", "temp2")]).value, + 41.0 + ); + assert_eq!( + find_one(&metrics, "node_hwmon_temp_celsius", &[("sensor", "temp3")]).value, + 40.0 + ); + assert!(matching(&metrics, "node_hwmon_temp_celsius", &[("sensor", "")]).is_empty()); + + // The human-readable text is published separately, keyed by chip and sensor, as + // Prometheus node_exporter does. Only the sensor that has a non-empty label file + // gets one. + let sensor_label = find_one( + &metrics, + "node_hwmon_sensor_label", + &[ + ("chip", chip), + ("sensor", "temp1"), + ("label", "Package id 0"), + ], + ); + assert_eq!(sensor_label.value, 1.0); + assert_eq!(sensor_label.unit, None); + // ...and only that one sensor gets an annotation + let labelled: Vec = matching(&metrics, "node_hwmon_sensor_label", &[]) + .iter() + .map(|m| label(m, "sensor").to_string()) + .collect(); + assert_eq!(labelled, ["temp1"]); + + // intrusion0_input is not a supported sensor kind and is skipped. + assert_eq!(matching(&metrics, "node_hwmon_in_volts", &[]).len(), 1); + } + + #[test] + fn test_collect_hwmon_same_model_chips_get_distinct_labels() { + let sys = tempfile::tempdir().unwrap(); + for (hwmon, device) in [("hwmon1", "nvme0"), ("hwmon2", "nvme1")] { + write(sys.path(), &format!("class/hwmon/{}/name", hwmon), "nvme\n"); + write( + sys.path(), + &format!("class/hwmon/{}/temp1_input", hwmon), + "38000\n", + ); + symlink( + format!("../../devices/pci0000:00/0000:00:1d.0/nvme/{}", device), + sys.path().join(format!("class/hwmon/{}/device", hwmon)), + ) + .unwrap(); + } + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_hwmon(&config_for(vec![Collector::Hwmon])); + + // Two chips of the same model must not collapse onto one attribute set: + // the `chip` label identifies the instance, `chip_name` the model. + assert_eq!(label_values(&metrics, "chip"), ["nvme0", "nvme1"]); + assert_eq!( + matching(&metrics, "node_hwmon_temp_celsius", &[("sensor", "temp1")]).len(), + 2 + ); + for chip in ["nvme0", "nvme1"] { + assert_eq!( + find_one( + &metrics, + "node_hwmon_temp_celsius", + &[("chip", chip), ("sensor", "temp1")] + ) + .value, + 38.0 + ); + find_one( + &metrics, + "node_hwmon_chip_names", + &[("chip", chip), ("chip_name", "nvme")], + ); + } + } + + /// The three ways a `chip` identity can be derived, in preference order. + /// + /// Only the first is exercised elsewhere; the fallbacks are what a host without + /// `device` symlinks (embedded platform drivers) actually takes. + /// The threshold metrics check their own names, so excluding the base reading must not + /// make them unreachable — an operator trimming cardinality still wants the limits. + #[test] + fn test_collect_hwmon_thresholds_survive_excluding_the_reading() { + let sys = tempfile::tempdir().unwrap(); + let hwmon = sys.path().join("class/hwmon/hwmon0"); + std::fs::create_dir_all(&hwmon).unwrap(); + std::fs::write(hwmon.join("name"), "coretemp\n").unwrap(); + std::fs::write(hwmon.join("temp1_input"), "42000\n").unwrap(); + std::fs::write(hwmon.join("temp1_max"), "84000\n").unwrap(); + std::fs::write(hwmon.join("temp1_crit"), "100000\n").unwrap(); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let config = NodeMetricsConfig { + collectors: vec![Collector::Hwmon], + sysfs_path: sys.path().to_str().unwrap().to_string(), + ..Default::default() + } + .with_exclude_filter("^node_hwmon_temp_celsius$") + .unwrap(); + + let metrics = collector.collect_hwmon(&config); + + assert!(matching(&metrics, "node_hwmon_temp_celsius", &[]).is_empty()); + assert_eq!( + find_one(&metrics, "node_hwmon_temp_max_celsius", &[]).value, + 84.0 + ); + assert_eq!( + find_one(&metrics, "node_hwmon_temp_crit_celsius", &[]).value, + 100.0 + ); + } + + #[test] + fn test_collect_hwmon_chip_identity_fallback_chain() { + let sys = tempfile::tempdir().unwrap(); + + // hwmon0: a `device` symlink, so the chip is the backing device name. + write(sys.path(), "class/hwmon/hwmon0/name", "coretemp\n"); + write(sys.path(), "class/hwmon/hwmon0/temp1_input", "42000\n"); + symlink( + "../../devices/platform/coretemp.0", + sys.path().join("class/hwmon/hwmon0/device"), + ) + .unwrap(); + + // hwmon1: no symlink, but a name file, so the chip combines the two. + write(sys.path(), "class/hwmon/hwmon1/name", "bq27xxx-battery\n"); + write(sys.path(), "class/hwmon/hwmon1/temp1_input", "38000\n"); + + // hwmon2: neither, so only the directory name is left to identify it. + write(sys.path(), "class/hwmon/hwmon2/temp1_input", "39000\n"); + + // hwmon3: the same model name as hwmon1 and no symlink either — the case that + // a name-only identity would collapse onto one attribute set. + write(sys.path(), "class/hwmon/hwmon3/name", "bq27xxx-battery\n"); + write(sys.path(), "class/hwmon/hwmon3/temp1_input", "40000\n"); + + let (_procfs, collector) = collector_for_sysfs(sys.path()); + let metrics = collector.collect_hwmon(&config_for(vec![Collector::Hwmon])); + + // Four chips, four distinct identities: the device name where there is one, the + // sanitised model name plus the hwmonN directory where there is a name file, and + // the bare directory name where there is neither. + assert_eq!( + label_values(&metrics, "chip"), + [ + "bq27xxx_battery_hwmon1", + "bq27xxx_battery_hwmon3", + "coretemp_0", + "hwmon2" + ] + ); + + for (chip, temp) in [ + ("coretemp_0", 42.0), + ("bq27xxx_battery_hwmon1", 38.0), + ("hwmon2", 39.0), + ("bq27xxx_battery_hwmon3", 40.0), + ] { + let reading = find_one( + &metrics, + "node_hwmon_temp_celsius", + &[("chip", chip), ("sensor", "temp1")], + ); + assert_eq!(reading.value, temp, "chip {}", chip); + assert_eq!(reading.unit.as_deref(), Some("Cel")); + assert_eq!(reading.metric_type, MetricType::Gauge); + } + + // The invariant that matters: two chips reporting the same model with no + // `device` symlink must still be distinguishable. Identical `chip` labels would + // make them one duplicate attribute set, and one chip's readings would be + // dropped in the OTLP conversion. + let same_model: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "node_hwmon_chip_names") + .filter(|m| label(m, "chip_name") == "bq27xxx-battery") + .map(|m| label(m, "chip")) + .collect(); + assert_eq!(same_model.len(), 2, "expected both same-model chips"); + assert_ne!( + same_model[0], same_model[1], + "two chips of the same model must not share a chip label" + ); + + // The model name is only published where one exists; hwmon2 has no name file, + // so it gets no annotation metric — but its readings are still reported above. + let mut chip_names: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "node_hwmon_chip_names") + .map(|m| label(m, "chip_name")) + .collect(); + chip_names.sort_unstable(); + chip_names.dedup(); + assert_eq!(chip_names, ["bq27xxx-battery", "coretemp"]); + assert!(matching(&metrics, "node_hwmon_chip_names", &[("chip", "hwmon2")]).is_empty()); + } +} diff --git a/src/receivers/node_metrics/collector/test_support.rs b/src/receivers/node_metrics/collector/test_support.rs new file mode 100644 index 00000000..bbffb581 --- /dev/null +++ b/src/receivers/node_metrics/collector/test_support.rs @@ -0,0 +1,149 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Fixture helpers shared by the collector unit tests. + +use crate::receivers::node_metrics::collector::CollectedMetric; +use crate::receivers::node_metrics::collector::SystemCollector; +use crate::receivers::node_metrics::config::Collector; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use std::path::Path; +use tempfile::TempDir; + +// --------------------------------------------------------------------- +// Fixture helpers +// +// Every collector takes its procfs/sysfs root as a parameter, so the whole +// module is testable from a fixture directory tree on any platform. +// --------------------------------------------------------------------- + +/// Write `contents` to `dir`/`rel`, creating any missing parent directories. +pub(super) fn write(dir: &Path, rel: &str, contents: &str) { + let path = dir.join(rel); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, contents).unwrap(); +} + +/// A default config restricted to `collectors`, with no metric filters. +pub(super) fn config_for(collectors: Vec) -> NodeMetricsConfig { + NodeMetricsConfig { + collectors, + ..Default::default() + } +} + +/// A collector reading `sysfs` as its sysfs root, with its procfs root pointed at an +/// empty fixture directory. +/// +/// For collectors that read sysfs only: passing the real `/proc` would make the test +/// depend on the host, and would read `/proc/stat` on every construction. +/// The returned [`TempDir`] must be kept alive for as long as the collector is used. +pub(super) fn collector_for_sysfs(sysfs: &Path) -> (TempDir, SystemCollector) { + let procfs = tempfile::tempdir().unwrap(); + let collector = SystemCollector::new(procfs.path().to_str().unwrap(), sysfs.to_str().unwrap()); + (procfs, collector) +} + +/// A collector whose procfs and sysfs roots are both an empty fixture directory. +/// +/// For collectors that read neither root — the textfile collector takes its directory +/// from the config — so the test touches nothing outside its own fixtures. +/// The returned [`TempDir`] must be kept alive for as long as the collector is used. +pub(super) fn collector_with_empty_roots() -> (TempDir, SystemCollector) { + let roots = tempfile::tempdir().unwrap(); + let path = roots.path().to_str().unwrap().to_string(); + let collector = SystemCollector::new(&path, &path); + (roots, collector) +} + +/// All metrics named `name` that carry every one of `labels`. +pub(super) fn matching<'a>( + metrics: &'a [CollectedMetric], + name: &str, + labels: &[(&str, &str)], +) -> Vec<&'a CollectedMetric> { + metrics + .iter() + .filter(|m| { + m.name == name + && labels + .iter() + .all(|(k, v)| m.labels.iter().any(|(mk, mv)| mk == k && mv == v)) + }) + .collect() +} + +/// The single metric named `name` carrying `labels`; panics otherwise. +pub(super) fn find_one<'a>( + metrics: &'a [CollectedMetric], + name: &str, + labels: &[(&str, &str)], +) -> &'a CollectedMetric { + let mut found = matching(metrics, name, labels); + assert_eq!( + found.len(), + 1, + "expected exactly one {} with labels {:?}, found {}", + name, + labels, + found.len() + ); + found.pop().unwrap() +} + +/// Value of label `key`, or `""` when the label is absent. +pub(super) fn label<'a>(metric: &'a CollectedMetric, key: &str) -> &'a str { + metric + .labels + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) + .unwrap_or_default() +} + +/// Sorted, deduplicated values of label `key` across `metrics`. +pub(super) fn label_values(metrics: &[CollectedMetric], key: &str) -> Vec { + let mut values: Vec = metrics.iter().map(|m| label(m, key).to_string()).collect(); + values.sort(); + values.dedup(); + values +} + +/// Total bytes the real `statfs(2)` reports for `path`. +pub(super) fn statfs_size_bytes(path: &str) -> f64 { + // Match the collector's choice of interface so the expected value is computed the same + // way on every target: Linux uses the LFS variant. + #[cfg(not(target_os = "linux"))] + use libc::{statfs, statfs as StatFs}; + #[cfg(target_os = "linux")] + use libc::{statfs64 as StatFs, statfs64 as statfs}; + + let mut stat: StatFs = unsafe { std::mem::zeroed() }; + let cpath = std::ffi::CString::new(path).unwrap(); + assert_eq!( + unsafe { statfs(cpath.as_ptr(), &mut stat) }, + 0, + "statfs({}) failed: {}", + path, + std::io::Error::last_os_error() + ); + stat.f_blocks as f64 * stat.f_bsize as f64 +} + +/// A trimmed but realistically shaped `/proc/stat`. +/// +/// `cpu2` is deliberately truncated after the `system` field, as a kernel +/// predating the later columns would report it. +pub(super) const STAT_FIXTURE: &str = "\ +cpu 3005 606 907 1200000 1500 180 210 24 2700 30 +cpu0 1000 200 300 400000 500 60 70 8 900 10 +cpu1 2000 400 600 800000 1000 120 140 16 1800 20 +cpu2 5 6 7 +intr 1234 0 0 +ctxt 987654321 +btime 1700000000 +processes 4242 +procs_running 3 +procs_blocked 1 +"; diff --git a/src/receivers/node_metrics/collector/textfile.rs b/src/receivers/node_metrics/collector/textfile.rs new file mode 100644 index 00000000..627b7f86 --- /dev/null +++ b/src/receivers/node_metrics/collector/textfile.rs @@ -0,0 +1,1223 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Textfile collector and Prometheus exposition-format parsing. +//! +//! Behaviour and the `node_textfile_*` semantics are derived from Prometheus +//! node_exporter — see the notice in this module's `mod.rs`. + +use crate::receivers::node_metrics::collector::{CollectedMetric, MetricType, SystemCollector}; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use std::collections::HashMap; +use std::fs; +use tracing::{debug, warn}; + +impl SystemCollector { + /// Collect custom metrics from Prometheus-format textfiles + /// + /// The textfile path can be either a directory (all `*.prom` files are read in + /// sorted filename order) or a single `.prom` file. + pub(super) fn collect_textfile(&self, config: &NodeMetricsConfig) -> Vec { + let mut metrics = Vec::new(); + + let textfile_path = match &config.textfile_directory { + Some(dir) => dir, + None => { + debug!("Textfile collector enabled but no directory configured"); + return metrics; + } + }; + + let path = std::path::Path::new(textfile_path); + + // Support both single-file and directory paths + let prom_files: Vec = if path.is_file() { + vec![path.to_path_buf()] + } else { + match fs::read_dir(path) { + Ok(entries) => { + let mut files: Vec = entries + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().and_then(|s| s.to_str()) == Some("prom")) + // Regular files only: a FIFO or socket named `foo.prom` would + // otherwise reach `read_to_string`, where `open(2)` blocks + // indefinitely with no writer and would wedge the scrape thread. + // `unwrap_or(true)` keeps an entry whose metadata cannot be read — + // a dangling symlink, say — so it is still reported through + // `node_textfile_scrape_error` rather than disappearing. + // `fs::metadata` follows symlinks, unlike `DirEntry::file_type`, + // so a valid symlinked `.prom` is collected. + .filter(|p| fs::metadata(p).map(|m| m.is_file()).unwrap_or(true)) + .collect(); + // Read in a deterministic order: the OTLP conversion takes a metric's + // description and unit from the first sample it sees, so traversal + // order would otherwise decide which file's HELP text wins, and would + // reorder data points between scrapes. + files.sort(); + files + } + Err(e) => { + // A mistyped or unreadable directory must not fail silently, otherwise + // the receiver just emits nothing forever. + warn!( + "Failed to read textfile path {} (set via --node-metrics-receiver-textfile-directory): {}", + textfile_path, e + ); + if config.should_include_metric("node_textfile_scrape_error") { + metrics.push(textfile_scrape_error(textfile_path.clone(), 1.0)); + } + return metrics; + } + } + }; + + let mut total_samples = 0usize; + let mut over_sample_limit = false; + + for path in &prom_files { + let filename = path + .file_name() + .map(|f| f.to_string_lossy().to_string()) + .unwrap_or_default(); + + // Expose the file's modification time so a stalled writer script is + // detectable even while its (stale) metrics keep being exported. + let metadata = fs::metadata(path).ok(); + if let Some(mtime) = metadata + .as_ref() + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + && config.should_include_metric("node_textfile_mtime_seconds") + { + metrics.push(CollectedMetric { + name: "node_textfile_mtime_seconds".to_string(), + value: mtime.as_secs_f64(), + labels: vec![("filename".to_string(), filename.clone())], + metric_type: MetricType::Gauge, + unit: Some("s".to_string()), + description: Some("Modification time of the textfile in seconds".to_string()), + }); + } + + // Guard against a runaway writer script: an unbounded file would be read + // into memory in full on every scrape interval. + if let Some(size) = metadata.as_ref().map(|m| m.len()) + && size > MAX_TEXTFILE_SIZE_BYTES + { + // Logged at debug, not warn: this repeats every scrape for as long as the + // file is oversized. `node_textfile_scrape_error` is the durable signal. + debug!( + "Skipping textfile {:?}: {} bytes exceeds the {} byte limit", + path, size, MAX_TEXTFILE_SIZE_BYTES + ); + if config.should_include_metric("node_textfile_scrape_error") { + metrics.push(textfile_scrape_error(filename, 1.0)); + } + continue; + } + + let content = match read_capped(path) { + Ok(c) => c, + Err(e) => { + // Debug, like the sibling failure paths: this repeats every scrape for + // as long as the file is unreadable, and `node_textfile_scrape_error` + // is the durable signal. + debug!("Failed to read textfile {:?}: {}", path, e); + if config.should_include_metric("node_textfile_scrape_error") { + metrics.push(textfile_scrape_error(filename, 1.0)); + } + continue; + } + }; + + // Track metric types and descriptions from comments, scoped per file. + // TYPE annotations determine counter vs gauge; conflicting types for + // the same metric across files coexist and the OTLP conversion keeps + // them as separate metrics (see `convert.rs`). + let mut described: std::collections::HashSet = std::collections::HashSet::new(); + let mut metric_types: HashMap = HashMap::new(); + let mut metric_descriptions: HashMap = HashMap::new(); + let mut parsed = Vec::new(); + let mut parse_failures = 0usize; + + // Parse the file + for line in content.lines() { + let line = line.trim(); + + // Skip empty lines + if line.is_empty() { + continue; + } + + // Parse HELP comments — preserve descriptions for OTLP. + // The exposition format allows any run of whitespace after the keyword. + if let Some(rest) = strip_comment_keyword(line, "HELP") { + if let Some((name, desc)) = rest.trim_start().split_once(char::is_whitespace) + && !name.is_empty() + { + metric_descriptions + .insert(name.to_string(), unescape_exposition(desc.trim())); + } + continue; + } + + // Parse TYPE comments + if let Some(rest) = strip_comment_keyword(line, "TYPE") { + if let Some((name, type_str)) = + rest.trim_start().split_once(char::is_whitespace) + && !name.is_empty() + { + let type_str = type_str.trim(); + let metric_type = match type_str { + "counter" => MetricType::Counter, + "gauge" => MetricType::Gauge, + // histogram/summary/untyped are not modelled; their component + // series are exported as gauges. + other => { + debug!( + "Textfile {}: unsupported metric type {:?} for {}, treating as gauge", + filename, other, name + ); + MetricType::Gauge + } + }; + metric_types.insert(name.to_string(), metric_type); + } + continue; + } + + // Skip other comments + if line.starts_with('#') { + continue; + } + + // Parse metric line + match parse_prometheus_line(line, &metric_types) { + Some(mut metric) => { + if !config.should_include_metric(&metric.name) { + continue; + } + // Attach HELP description if available + // Attach the HELP text to the first sample of each metric only: the + // OTLP conversion takes the description from the first member of a + // group, so cloning it onto every sample would hold hundreds of + // thousands of copies of the same string for a large file. + if let Some(desc) = metric_descriptions.get(&metric.name) + && described.insert(metric.name.clone()) + { + metric.description = Some(desc.clone()); + } + parsed.push(metric); + } + None => { + // A malformed line must not disappear without a trace, otherwise a + // broken writer script is indistinguishable from an empty file. + parse_failures += 1; + debug!("Textfile {}: skipping malformed line {:?}", filename, line); + } + } + } + + if parse_failures > 0 { + // Debug rather than warn: a static file with a bad line would otherwise log + // on every scrape forever. `node_textfile_scrape_error` carries the signal. + debug!( + "Textfile {}: skipped {} malformed line(s)", + filename, parse_failures + ); + } + + if config.should_include_metric("node_textfile_scrape_error") { + let value = if parse_failures > 0 { 1.0 } else { 0.0 }; + metrics.push(textfile_scrape_error(filename.clone(), value)); + } + + if parsed.len() + total_samples > MAX_TEXTFILE_SAMPLES { + let room = MAX_TEXTFILE_SAMPLES.saturating_sub(total_samples); + warn!( + "Textfile {}: keeping {} of {} samples; the {} sample per-scrape limit \ + was reached", + filename, + room, + parsed.len(), + MAX_TEXTFILE_SAMPLES + ); + parsed.truncate(room); + over_sample_limit = true; + } + total_samples += parsed.len(); + metrics.append(&mut parsed); + + if over_sample_limit { + break; + } + } + + metrics + } +} + +/// Maximum number of samples taken from the textfile directory in one scrape +const MAX_TEXTFILE_SAMPLES: usize = 100_000; + +/// Read a textfile, enforcing the size cap during the read +/// +/// The pre-read `stat` is only advisory: a file can grow between the stat and the read, and +/// a non-regular file reports a length of zero. Reading through a bounded reader means the +/// cap holds regardless. +fn read_capped(path: &std::path::Path) -> std::io::Result { + use std::io::Read; + + let mut buf = String::new(); + fs::File::open(path)? + // One byte over the cap, so an oversized file is detectable rather than silently + // truncated mid-sample. + .take(MAX_TEXTFILE_SIZE_BYTES + 1) + .read_to_string(&mut buf)?; + + if buf.len() as u64 > MAX_TEXTFILE_SIZE_BYTES { + return Err(std::io::Error::other(format!( + "textfile exceeds the {} byte limit", + MAX_TEXTFILE_SIZE_BYTES + ))); + } + Ok(buf) +} + +/// Maximum size of a single `.prom` textfile that will be read +const MAX_TEXTFILE_SIZE_BYTES: u64 = 10 * 1024 * 1024; + +/// Build the `node_textfile_scrape_error` metric for a given file +fn textfile_scrape_error(filename: String, value: f64) -> CollectedMetric { + CollectedMetric { + name: "node_textfile_scrape_error".to_string(), + value, + labels: vec![("filename".to_string(), filename)], + metric_type: MetricType::Gauge, + unit: None, + description: Some("1 if there was an error scraping the textfile, 0 otherwise".to_string()), + } +} + +/// Strip a `# ` comment prefix, returning the remainder. +/// +/// The exposition format allows any run of whitespace after `#` and after the keyword, +/// so `#HELP`, `# HELP` and `# HELP` are all accepted. Returns `None` when the line is +/// not this kind of comment, so that plain comments fall through untouched. +fn strip_comment_keyword<'a>(line: &'a str, keyword: &str) -> Option<&'a str> { + let rest = line.strip_prefix('#')?.trim_start(); + let rest = rest.strip_prefix(keyword)?; + // The keyword must be followed by whitespace, so `# HELPfoo` is not a HELP line + if rest.starts_with(char::is_whitespace) { + Some(rest) + } else { + None + } +} + +/// Decode the escape sequences the Prometheus exposition format allows in HELP text +fn unescape_exposition(s: &str) -> String { + if !s.contains('\\') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('n') => out.push('\n'), + Some('\\') => out.push('\\'), + // Any other escape is passed through verbatim, as the format leaves it undefined + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + } + } + out +} + +/// Parse a single Prometheus exposition format line +/// Format: metric_name[{label="value",...}] value [timestamp] +fn parse_prometheus_line( + line: &str, + metric_types: &HashMap, +) -> Option { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + + // Find the metric name and optional labels + let (name, labels_str, value_str) = if let Some(brace_start) = line.find('{') { + // Has labels — find closing '}' while respecting quoted strings + let brace_end = find_closing_brace(&line[brace_start + 1..])?; + let brace_end = brace_start + 1 + brace_end; + // Whitespace between the name and `{` is tolerated by the exposition format + let name = line[..brace_start].trim_end(); + let labels_str = Some(&line[brace_start + 1..brace_end]); + let rest = line[brace_end + 1..].trim(); + (name, labels_str, rest) + } else { + // No labels - split on whitespace + let mut parts = line.splitn(2, char::is_whitespace); + let name = parts.next()?; + let rest = parts.next().unwrap_or("").trim(); + (name, None, rest) + }; + + // Reject names that are not valid metric identifiers — `.prom` content is + // user-supplied and flows straight into the OTLP metric name. + if !is_valid_metric_name(name) { + return None; + } + + // Parse the value, ignoring the optional trailing timestamp: as in Prometheus + // `node_exporter`, textfile samples are stamped at scrape time. + // `f64::from_str` already accepts the format's `+Inf`, `-Inf` and `NaN` spellings. + let value_str = value_str.split_whitespace().next()?; + let value: f64 = value_str.parse().ok()?; + + // Parse labels if present + let mut labels: Vec<(String, String)> = Vec::new(); + if let Some(labels_str) = labels_str { + for part in split_labels(labels_str) { + // Any malformed label fails the whole line rather than being dropped: losing a + // label changes the series identity, which would collapse distinct samples onto + // one attribute set with no diagnostic. The caller counts this as a parse + // failure and raises `node_textfile_scrape_error`. + let (key, val) = parse_label(part)?; + + // `.prom` content is user-supplied and flows straight into OTLP attribute keys, + // so apply the same grammar check as the metric name. + if !is_valid_label_name(&key) { + return None; + } + // Keep the first occurrence of a repeated key — two attributes sharing a key + // would make the data point invalid, and first-wins preserves identity. + if labels.iter().any(|(k, _)| *k == key) { + continue; + } + labels.push((key, val)); + } + } + + // Determine metric type (default to gauge) + let metric_type = metric_types.get(name).copied().unwrap_or(MetricType::Gauge); + + Some(CollectedMetric { + name: name.to_string(), + value, + labels, + metric_type, + unit: None, + description: None, + }) +} + +/// Check that a name matches the Prometheus label name grammar `[a-zA-Z_][a-zA-Z0-9_]*` +fn is_valid_label_name(name: &str) -> bool { + let mut chars = name.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' => {} + _ => return false, + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +/// Check that a name matches the Prometheus metric name grammar `[a-zA-Z_:][a-zA-Z0-9_:]*` +fn is_valid_metric_name(name: &str) -> bool { + let mut chars = name.chars(); + match chars.next() { + Some(c) if c.is_ascii_alphabetic() || c == '_' || c == ':' => {} + _ => return false, + } + chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == ':') +} + +/// Find the closing `}` in a label block, skipping over quoted strings and escapes. +/// Input starts right after the opening `{`. Returns the byte offset of `}` within `s`. +fn find_closing_brace(s: &str) -> Option { + let mut in_quotes = false; + let mut prev_backslash = false; + for (i, c) in s.char_indices() { + match c { + '\\' if in_quotes => { + prev_backslash = !prev_backslash; + continue; + } + '"' if !prev_backslash => in_quotes = !in_quotes, + '}' if !in_quotes => return Some(i), + _ => {} + } + prev_backslash = false; + } + None +} + +/// Split label string by commas, respecting quoted values and escape sequences +fn split_labels(labels_str: &str) -> Vec<&str> { + let mut parts = Vec::new(); + let mut start = 0; + let mut in_quotes = false; + let mut prev_backslash = false; + + for (i, c) in labels_str.char_indices() { + match c { + '\\' if in_quotes => { + prev_backslash = !prev_backslash; + continue; + } + '"' if !prev_backslash => in_quotes = !in_quotes, + ',' if !in_quotes => { + parts.push(&labels_str[start..i]); + start = i + 1; + } + _ => {} + } + prev_backslash = false; + } + + // Add the last part + if start < labels_str.len() { + parts.push(&labels_str[start..]); + } + + parts +} + +/// Parse a single label: key="value" +fn parse_label(label: &str) -> Option<(String, String)> { + let label = label.trim(); + let eq_pos = label.find('=')?; + let key = label[..eq_pos].trim(); + let val = label[eq_pos + 1..].trim(); + + // An empty key would become an empty OTLP attribute key + if key.is_empty() { + return None; + } + + // Remove exactly one surrounding quote from each end + let val = if val.starts_with('"') && val.ends_with('"') && val.len() >= 2 { + &val[1..val.len() - 1] + } else { + val + }; + + // Handle escape sequences in a single pass (avoids NUL-byte placeholder issues) + let mut result = String::with_capacity(val.len()); + let mut chars = val.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + match chars.next() { + Some('\\') => result.push('\\'), + Some('"') => result.push('"'), + Some('n') => result.push('\n'), + Some(other) => { + result.push('\\'); + result.push(other); + } + None => result.push('\\'), + } + } else { + result.push(c); + } + } + + Some((key.to_string(), result)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::receivers::node_metrics::collector::test_support::*; + use crate::receivers::node_metrics::config::Collector; + use std::fs::File; + use std::os::unix::fs::symlink; + + // --------------------------------------------------------------------- + // Textfile collector + // --------------------------------------------------------------------- + + #[test] + fn test_collect_textfile_directory_contents() { + let tmp = tempfile::tempdir().unwrap(); + // Created in the exact reverse of sorted filename order, so a collector that + // dropped the sort could only pass by accident: the OTLP conversion takes a + // metric's description and unit from the first sample it sees, and reorders + // data points between scrapes if traversal order decides which file wins. + std::fs::write(tmp.path().join("z.prom"), "shared_metric{job=\"z\"} 40\n").unwrap(); + std::fs::write( + tmp.path().join("m.prom"), + "\ +# HELP shared_metric Documented in m.prom +shared_metric{job=\"m\"} 30 +", + ) + .unwrap(); + std::fs::write( + tmp.path().join("b.prom"), + "\ +# HELP other_metric Documented in b.prom +shared_metric{job=\"b\"} 20 +other_metric 3.5 +", + ) + .unwrap(); + std::fs::write( + tmp.path().join("a.prom"), + "\ +# HELP shared_metric Documented in a.prom +# TYPE shared_metric counter +shared_metric{job=\"a\"} 10 +this line is not valid exposition format +", + ) + .unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let config = NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(tmp.path().to_str().unwrap().to_string()), + ..Default::default() + }; + + let metrics = collector.collect_textfile(&config); + + // Values, labels, types and HELP text all come from the file. + let from_a = find_one(&metrics, "shared_metric", &[("job", "a")]); + assert_eq!(from_a.value, 10.0); + assert_eq!(from_a.metric_type, MetricType::Counter); + assert_eq!(from_a.description.as_deref(), Some("Documented in a.prom")); + + // TYPE and HELP are scoped to the file that declares them. + let from_b = find_one(&metrics, "shared_metric", &[("job", "b")]); + assert_eq!(from_b.value, 20.0); + assert_eq!(from_b.metric_type, MetricType::Gauge); + assert_eq!(from_b.description, None); + + let other = find_one(&metrics, "other_metric", &[]); + assert_eq!(other.value, 3.5); + assert_eq!(other.description.as_deref(), Some("Documented in b.prom")); + + let from_m = find_one(&metrics, "shared_metric", &[("job", "m")]); + assert_eq!(from_m.value, 30.0); + assert_eq!(from_m.description.as_deref(), Some("Documented in m.prom")); + + let from_z = find_one(&metrics, "shared_metric", &[("job", "z")]); + assert_eq!(from_z.value, 40.0); + assert_eq!(from_z.description, None); + + // The malformed line marks only its own file as failed. + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "a.prom")] + ) + .value, + 1.0 + ); + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "b.prom")] + ) + .value, + 0.0 + ); + + // Modification times are exported so a stalled writer is detectable, and + // the files are visited in sorted filename order, not creation order. + let mtimes: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "node_textfile_mtime_seconds") + .map(|m| label(m, "filename")) + .collect(); + assert_eq!(mtimes, ["a.prom", "b.prom", "m.prom", "z.prom"]); + + // The samples themselves are emitted in that same order, so the description + // and unit the OTLP conversion keeps for `shared_metric` are always a.prom's. + let shared_order: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "shared_metric") + .map(|m| label(m, "job")) + .collect(); + assert_eq!(shared_order, ["a", "b", "m", "z"]); + + assert!( + find_one( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "a.prom")] + ) + .value + > 0.0 + ); + } + + #[test] + fn test_textfile_help_descriptions() { + let tmp = tempfile::tempdir().unwrap(); + let prom = tmp.path().join("test.prom"); + std::fs::write( + &prom, + "\ +# HELP my_gauge A helpful description for the gauge +# TYPE my_gauge gauge +my_gauge 42 +# HELP my_counter Total number of things +# TYPE my_counter counter +my_counter 100 +no_help_metric 7 +", + ) + .unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let config = NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(prom.to_str().unwrap().to_string()), + ..Default::default() + }; + + let metrics = collector.collect_textfile(&config); + + let find = |name: &str| metrics.iter().find(|m| m.name == name); + + // Metrics with HELP comments should have descriptions + let gauge = find("my_gauge").expect("my_gauge not found"); + assert_eq!( + gauge.description.as_deref(), + Some("A helpful description for the gauge") + ); + assert_eq!(gauge.metric_type, MetricType::Gauge); + + let counter = find("my_counter").expect("my_counter not found"); + assert_eq!( + counter.description.as_deref(), + Some("Total number of things") + ); + assert_eq!(counter.metric_type, MetricType::Counter); + + // Metric without HELP should have no description + let no_help = find("no_help_metric").expect("no_help_metric not found"); + assert_eq!(no_help.description, None); + } + + #[test] + fn test_textfile_single_file_path() { + let tmp = tempfile::tempdir().unwrap(); + let prom = tmp.path().join("single.prom"); + std::fs::write(&prom, "single_metric 99\n").unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let config = NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(prom.to_str().unwrap().to_string()), + ..Default::default() + }; + + let metrics = collector.collect_textfile(&config); + let found = metrics.iter().find(|m| m.name == "single_metric"); + assert!( + found.is_some(), + "single_metric not found in {:?}", + metrics.iter().map(|m| &m.name).collect::>() + ); + assert_eq!(found.unwrap().value, 99.0); + } + + #[test] + fn test_textfile_directory_path() { + let tmp = tempfile::tempdir().unwrap(); + std::fs::write(tmp.path().join("a.prom"), "metric_a 1\n").unwrap(); + std::fs::write(tmp.path().join("b.prom"), "metric_b 2\n").unwrap(); + std::fs::write(tmp.path().join("ignored.txt"), "not_a_metric 3\n").unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let config = NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(tmp.path().to_str().unwrap().to_string()), + ..Default::default() + }; + + let metrics = collector.collect_textfile(&config); + let names: Vec<&str> = metrics.iter().map(|m| m.name.as_str()).collect(); + + assert!( + names.contains(&"metric_a"), + "metric_a not found in {:?}", + names + ); + assert!( + names.contains(&"metric_b"), + "metric_b not found in {:?}", + names + ); + assert!( + !names.contains(&"not_a_metric"), + ".txt file should be ignored" + ); + } + + /// The textfile config for a `.prom` file or a directory of them. + fn textfile_config(path: &str) -> NodeMetricsConfig { + NodeMetricsConfig { + collectors: vec![Collector::Textfile], + textfile_directory: Some(path.to_string()), + ..Default::default() + } + } + + /// A `.prom` that cannot be read — a dangling symlink is the realistic case — must be + /// reported, not silently skipped, or a deployment whose symlink target was garbage + /// collected loses metrics with no signal at all. + #[test] + fn test_textfile_dangling_symlink_is_reported_as_an_error() { + let tmp = tempfile::tempdir().unwrap(); + std::os::unix::fs::symlink( + tmp.path().join("no-such-target.prom"), + tmp.path().join("dangling.prom"), + ) + .unwrap(); + write(tmp.path(), "good.prom", "good_metric 1\n"); + + let collector = collector_with_empty_roots(); + let metrics = collector + .1 + .collect_textfile(&textfile_config(tmp.path().to_str().unwrap())); + + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "dangling.prom")] + ) + .value, + 1.0 + ); + // A dangling link has no mtime to report, and the healthy file is unaffected. + assert!( + matching( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "dangling.prom")] + ) + .is_empty() + ); + assert_eq!(find_one(&metrics, "good_metric", &[]).value, 1.0); + } + + /// A FIFO named `*.prom` must never be opened: `open(2)` on one with no writer blocks + /// forever, which would wedge the scrape thread and stop all node metrics. + #[test] + fn test_textfile_fifo_is_skipped_without_reading_it() { + let tmp = tempfile::tempdir().unwrap(); + let fifo = tmp.path().join("pipe.prom"); + let cpath = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap(); + // SAFETY: mkfifo on a path inside our own temp directory + let rc = unsafe { libc::mkfifo(cpath.as_ptr(), 0o600) }; + assert_eq!(rc, 0, "could not create fifo for the test"); + write(tmp.path(), "plain.prom", "plain_metric 7\n"); + + let collector = collector_with_empty_roots(); + // If the FIFO were opened this call would never return. + let metrics = collector + .1 + .collect_textfile(&textfile_config(tmp.path().to_str().unwrap())); + + assert_eq!(find_one(&metrics, "plain_metric", &[]).value, 7.0); + // Skipped silently: it is not a textfile at all, so it is not an error either. + assert!( + matching( + &metrics, + "node_textfile_scrape_error", + &[("filename", "pipe.prom")] + ) + .is_empty() + ); + } + + /// The exposition format allows any whitespace run after `#`, but the keyword itself + /// must be followed by whitespace — `# HELPfoo` is an ordinary comment. + #[test] + fn test_comment_keyword_whitespace_handling() { + let mut types = HashMap::new(); + types.insert("m".to_string(), MetricType::Counter); + + assert_eq!( + strip_comment_keyword("#HELP m desc", "HELP"), + Some(" m desc") + ); + assert_eq!( + strip_comment_keyword("# TYPE m counter", "TYPE"), + Some(" m counter") + ); + assert_eq!(strip_comment_keyword("# HELPfoo m desc", "HELP"), None); + assert_eq!(strip_comment_keyword("# TYPEfoo m counter", "TYPE"), None); + assert_eq!(strip_comment_keyword("# just a comment", "HELP"), None); + assert_eq!(strip_comment_keyword("not a comment", "HELP"), None); + assert_eq!(strip_comment_keyword("#", "HELP"), None); + } + + /// `.prom` content is user-supplied, so label keys get the same grammar check as metric + /// names. An invalid key fails the line — dropping just the label would change the + /// series identity and silently collapse distinct samples onto one attribute set. + #[test] + fn test_invalid_label_key_rejects_the_line_and_duplicates_are_deduped() { + let types = HashMap::new(); + + assert!(parse_prometheus_line(r#"m{bad-key="1"} 5"#, &types).is_none()); + assert!(parse_prometheus_line(r#"m{="1"} 5"#, &types).is_none()); + assert!(parse_prometheus_line(r#"m{1abc="1"} 5"#, &types).is_none()); + + // A colon is legal in a metric name but not in a label name + assert!(parse_prometheus_line(r#"m{a:b="1"} 5"#, &types).is_none()); + + // First occurrence of a repeated key wins, and the line still parses + let metric = parse_prometheus_line(r#"m{ok="first",ok="second"} 5"#, &types).unwrap(); + assert_eq!(metric.labels, vec![("ok".to_string(), "first".to_string())]); + } + + #[test] + fn test_textfile_oversized_file_is_skipped_and_flagged() { + let tmp = tempfile::tempdir().unwrap(); + + // A runaway writer script: reading this on every scrape interval would pull + // the whole file into memory. Created sparsely — only the first line is + // written, then the file is extended one byte past the cap. + let big = tmp.path().join("big.prom"); + std::fs::write(&big, "big_metric 1\n").unwrap(); + File::options() + .write(true) + .open(&big) + .unwrap() + .set_len(MAX_TEXTFILE_SIZE_BYTES + 1) + .unwrap(); + std::fs::write(tmp.path().join("small.prom"), "small_metric 2\n").unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let metrics = collector.collect_textfile(&textfile_config(tmp.path().to_str().unwrap())); + + // The oversized file is not parsed at all, and says so. + assert!(matching(&metrics, "big_metric", &[]).is_empty()); + let error = find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "big.prom")], + ); + assert_eq!(error.value, 1.0); + assert_eq!(error.metric_type, MetricType::Gauge); + assert_eq!(error.unit, None); + assert_eq!(error.labels.len(), 1); + + // Its modification time is still reported, so a writer stuck emitting an + // oversized file stays observable. + find_one( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "big.prom")], + ); + + // Skipping one file does not abandon the rest of the directory. + assert_eq!(find_one(&metrics, "small_metric", &[]).value, 2.0); + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "small.prom")] + ) + .value, + 0.0 + ); + } + + #[test] + fn test_textfile_missing_directory_reports_error_against_configured_path() { + let tmp = tempfile::tempdir().unwrap(); + let missing = tmp.path().join("no-such-directory"); + let configured = missing.to_str().unwrap().to_string(); + + let (_roots, collector) = collector_with_empty_roots(); + let metrics = collector.collect_textfile(&textfile_config(&configured)); + + // A mistyped directory must not fail silently, or the receiver just emits + // nothing forever. + assert_eq!(metrics.len(), 1); + let error = &metrics[0]; + assert_eq!(error.name, "node_textfile_scrape_error"); + assert_eq!(error.value, 1.0); + assert_eq!(error.metric_type, MetricType::Gauge); + assert_eq!(error.unit, None); + + // Deliberately asymmetric with the per-file errors, which carry a bare + // filename: there is no file here, so the configured path is the only thing + // that identifies what went wrong. + assert_eq!(error.labels, vec![("filename".to_string(), configured)]); + } + + #[test] + fn test_textfile_unsupported_type_falls_back_to_gauge() { + let tmp = tempfile::tempdir().unwrap(); + let prom = tmp.path().join("types.prom"); + std::fs::write( + &prom, + "\ +# TYPE my_histogram histogram +my_histogram 5 +# TYPE my_summary summary +my_summary 6 +# TYPE my_untyped untyped +my_untyped 7 +# TYPE my_counter counter +my_counter 8 +", + ) + .unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let metrics = collector.collect_textfile(&textfile_config(prom.to_str().unwrap())); + + // histogram, summary and untyped are not modelled; their series are exported + // as gauges rather than being dropped or promoted to counters. + for (name, value) in [ + ("my_histogram", 5.0), + ("my_summary", 6.0), + ("my_untyped", 7.0), + ] { + let metric = find_one(&metrics, name, &[]); + assert_eq!(metric.metric_type, MetricType::Gauge, "{}", name); + assert_eq!(metric.value, value, "{}", name); + } + + // A recognised type in the same file is still honoured. + let counter = find_one(&metrics, "my_counter", &[]); + assert_eq!(counter.metric_type, MetricType::Counter); + assert_eq!(counter.value, 8.0); + + // An unsupported TYPE is not a parse failure. + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "types.prom")] + ) + .value, + 0.0 + ); + } + + #[test] + fn test_textfile_symlinked_prom_file_is_collected() { + // The real file lives outside the scanned directory, as it does when a + // deployment symlinks generated metrics into place. + let elsewhere = tempfile::tempdir().unwrap(); + let real = elsewhere.path().join("generated.prom"); + std::fs::write(&real, "# TYPE linked_metric counter\nlinked_metric 5\n").unwrap(); + + let tmp = tempfile::tempdir().unwrap(); + symlink(&real, tmp.path().join("linked.prom")).unwrap(); + std::fs::write(tmp.path().join("plain.prom"), "plain_metric 1\n").unwrap(); + // A *directory* named like a textfile is still not a textfile. + std::fs::create_dir(tmp.path().join("adirectory.prom")).unwrap(); + + let (_roots, collector) = collector_with_empty_roots(); + let metrics = collector.collect_textfile(&textfile_config(tmp.path().to_str().unwrap())); + + // The scan resolves symlinks, so the link is read like any other file. + let linked = find_one(&metrics, "linked_metric", &[]); + assert_eq!(linked.value, 5.0); + assert_eq!(linked.metric_type, MetricType::Counter); + assert_eq!( + find_one( + &metrics, + "node_textfile_scrape_error", + &[("filename", "linked.prom")] + ) + .value, + 0.0 + ); + // The link's own name is what identifies it, not the target's. + find_one( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "linked.prom")], + ); + assert!( + matching( + &metrics, + "node_textfile_mtime_seconds", + &[("filename", "generated.prom")] + ) + .is_empty() + ); + + assert_eq!(find_one(&metrics, "plain_metric", &[]).value, 1.0); + + // Exactly two files were scanned: the directory named `adirectory.prom` is + // not reported as an unreadable textfile. + let mut scanned: Vec<&str> = metrics + .iter() + .filter(|m| m.name == "node_textfile_scrape_error") + .map(|m| label(m, "filename")) + .collect(); + scanned.sort_unstable(); + assert_eq!(scanned, ["linked.prom", "plain.prom"]); + } + + // --------------------------------------------------------------------- + // Prometheus exposition-format parsing + // --------------------------------------------------------------------- + + #[test] + fn test_parse_prometheus_line_simple() { + let types = HashMap::new(); + let metric = parse_prometheus_line("my_metric 42.5", &types).unwrap(); + assert_eq!(metric.name, "my_metric"); + assert_eq!(metric.value, 42.5); + assert!(metric.labels.is_empty()); + assert_eq!(metric.metric_type, MetricType::Gauge); + } + + #[test] + fn test_parse_prometheus_line_with_labels() { + let types = HashMap::new(); + let metric = parse_prometheus_line( + "http_requests_total{method=\"GET\",path=\"/api\"} 1234", + &types, + ) + .unwrap(); + assert_eq!(metric.name, "http_requests_total"); + assert_eq!(metric.value, 1234.0); + let find_label = |key: &str| { + metric + .labels + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.clone()) + }; + assert_eq!(find_label("method"), Some("GET".to_string())); + assert_eq!(find_label("path"), Some("/api".to_string())); + } + + #[test] + fn test_parse_prometheus_line_with_type() { + let mut types = HashMap::new(); + types.insert("my_counter".to_string(), MetricType::Counter); + + let metric = parse_prometheus_line("my_counter 100", &types).unwrap(); + assert_eq!(metric.metric_type, MetricType::Counter); + } + + #[test] + fn test_parse_prometheus_line_special_values() { + let types = HashMap::new(); + + let inf = parse_prometheus_line("metric_inf +Inf", &types).unwrap(); + assert!(inf.value.is_infinite() && inf.value.is_sign_positive()); + + let neg_inf = parse_prometheus_line("metric_neginf -Inf", &types).unwrap(); + assert!(neg_inf.value.is_infinite() && neg_inf.value.is_sign_negative()); + + let nan = parse_prometheus_line("metric_nan NaN", &types).unwrap(); + assert!(nan.value.is_nan()); + } + + #[test] + fn test_parse_prometheus_line_with_timestamp() { + let types = HashMap::new(); + // Timestamp should be ignored + let metric = parse_prometheus_line("my_metric 42.5 1395066363000", &types).unwrap(); + assert_eq!(metric.name, "my_metric"); + assert_eq!(metric.value, 42.5); + } + + #[test] + fn test_parse_prometheus_line_escaped_label() { + let types = HashMap::new(); + let metric = + parse_prometheus_line("metric{label=\"value with \\\"quotes\\\"\"} 1", &types).unwrap(); + let label_val = metric + .labels + .iter() + .find(|(k, _)| k == "label") + .map(|(_, v)| v.clone()); + assert_eq!(label_val, Some("value with \"quotes\"".to_string())); + } + + #[test] + fn test_parse_prometheus_line_brace_in_label_value() { + let types = HashMap::new(); + let metric = parse_prometheus_line(r#"metric{label="a}b"} 42"#, &types).unwrap(); + assert_eq!(metric.name, "metric"); + assert_eq!(metric.value, 42.0); + let label_val = metric + .labels + .iter() + .find(|(k, _)| k == "label") + .map(|(_, v)| v.clone()); + assert_eq!(label_val, Some("a}b".to_string())); + } + + #[test] + fn test_parse_prometheus_line_empty() { + let types = HashMap::new(); + assert!(parse_prometheus_line("", &types).is_none()); + assert!(parse_prometheus_line("# comment", &types).is_none()); + } + + #[test] + fn test_split_labels() { + let labels = split_labels("method=\"GET\",path=\"/api\""); + assert_eq!(labels.len(), 2); + assert_eq!(labels[0], "method=\"GET\""); + assert_eq!(labels[1], "path=\"/api\""); + } + + #[test] + fn test_split_labels_with_comma_in_value() { + let labels = split_labels("label=\"value,with,commas\",other=\"ok\""); + assert_eq!(labels.len(), 2); + assert_eq!(labels[0], "label=\"value,with,commas\""); + assert_eq!(labels[1], "other=\"ok\""); + } + + #[test] + fn test_split_labels_with_escaped_quotes() { + let labels = split_labels(r#"method="GET",path="/foo\"bar""#); + assert_eq!(labels.len(), 2); + assert_eq!(labels[0], r#"method="GET""#); + assert_eq!(labels[1], r#"path="/foo\"bar""#); + } + + #[test] + fn test_split_labels_with_escaped_backslash_before_quote() { + // \\\" in the source: escaped backslash followed by escaped quote + let labels = split_labels(r#"a="x\\\"y",b="ok""#); + assert_eq!(labels.len(), 2); + assert_eq!(labels[0], r#"a="x\\\"y""#); + assert_eq!(labels[1], r#"b="ok""#); + } + + #[test] + fn test_parse_label_escaped_backslash_before_n() { + // \\n in exposition format = literal backslash followed by 'n' + let result = parse_label(r#"key="a\\nb""#).unwrap(); + assert_eq!(result.0, "key"); + assert_eq!(result.1, "a\\nb"); + } + + #[test] + fn test_parse_label_unknown_escape() { + // Unknown escape sequences should preserve the backslash + let result = parse_label(r#"key="a\tb""#).unwrap(); + assert_eq!(result.1, "a\\tb"); + } +} diff --git a/src/receivers/node_metrics/collector/util.rs b/src/receivers/node_metrics/collector/util.rs new file mode 100644 index 00000000..2260063c --- /dev/null +++ b/src/receivers/node_metrics/collector/util.rs @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Path, device-name and `utsname` field helpers shared by the collectors. +//! +//! The virtual-device and partition exclusion heuristics are derived from Prometheus +//! node_exporter — see the notice in this module's `mod.rs`. + +/// Check if `path` matches a prefix with path-boundary awareness. +/// +/// Returns `true` when `path` equals `prefix` exactly or starts with `prefix` +/// followed by `/`. This prevents `/dev` from matching `/developer`. +/// +/// A trailing `/` on the prefix is ignored, so `/dev` and `/dev/` behave alike. +pub(crate) fn matches_path_prefix(path: &str, prefix: &str) -> bool { + // "/" trims to "", and the root prefix matches every absolute path + let trimmed = prefix.trim_end_matches('/'); + if trimmed.is_empty() { + return prefix.starts_with('/') && path.starts_with('/'); + } + path == trimmed + || path.starts_with(trimmed) && path.as_bytes().get(trimmed.len()) == Some(&b'/') +} + +/// Check if a device name is a virtual block device that should not be reported +/// +/// Covers the `z?ram`, `loop` and `fd` families of node_exporter's default device +/// exclusion (its full default also excludes partitions, which `is_partition` handles). +pub(super) fn is_virtual_disk(device: &str) -> bool { + for prefix in &["zram", "ram", "loop", "fd"] { + if let Some(rest) = device.strip_prefix(prefix) + && !rest.is_empty() + && rest.chars().all(|c| c.is_ascii_digit()) + { + return true; + } + } + false +} + +/// Check if a device name looks like a partition +/// +/// This is a name heuristic. Partitions are excluded from disk I/O metrics so their +/// counters are not double-counted alongside the whole disk that contains them. +/// +/// Deliberately stricter than node_exporter, whose default `p` rule applies only +/// to NVMe: this also treats `mmcblk0p1`, `md127p1`, `nbd0p1`, `zd16p1`, `pmem0p1` and +/// `dasda1` as partitions, so those per-partition series are not reported. +pub(super) fn is_partition(device: &str) -> bool { + // Devices whose whole-disk name ends in a digit delimit the partition with `p`: + // nvme0n1p1, mmcblk0p1, md127p1, nbd0p1, zd16p1, pmem0p1, loop0p1. + if let Some((disk, part)) = device.rsplit_once('p') + && !part.is_empty() + && part.chars().all(|c| c.is_ascii_digit()) + && disk.ends_with(|c: char| c.is_ascii_digit()) + { + return true; + } + + // SCSI/SATA/virtio/s390 partitions: sda1, vda1, xvda1, hda1, sdaa1, dasda1. + // After the prefix, one or more lowercase letters identify the disk, + // and any trailing digits indicate a partition number. + for prefix in &["sd", "vd", "xvd", "hd", "dasd"] { + if let Some(rest) = device.strip_prefix(prefix) { + let letter_count = rest.chars().take_while(|c| c.is_ascii_lowercase()).count(); + if letter_count == 0 { + continue; + } + let after_letters = &rest[letter_count..]; + if !after_letters.is_empty() && after_letters.chars().all(|c| c.is_ascii_digit()) { + return true; + } + } + } + + false +} + +/// Decode octal escape sequences from /proc/mounts paths. +/// +/// The kernel encodes space as `\040`, tab as `\011`, newline as `\012`, +/// and backslash as `\134` in device and mount-point fields. +pub(super) fn decode_mount_path(s: &str) -> String { + // Fast path: no escapes present (common case) + if !s.contains('\\') { + return s.to_string(); + } + let bytes = s.as_bytes(); + let mut result = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'\\' + && i + 3 < bytes.len() + && (b'0'..=b'7').contains(&bytes[i + 1]) + && (b'0'..=b'7').contains(&bytes[i + 2]) + && (b'0'..=b'7').contains(&bytes[i + 3]) + { + let val = (bytes[i + 1] - b'0') as u16 * 64 + + (bytes[i + 2] - b'0') as u16 * 8 + + (bytes[i + 3] - b'0') as u16; + // Three octal digits can represent up to \777 = 511, so + // guard against values that don't fit in a single byte. + // The kernel only emits valid byte values, but be safe. + if val <= 255 { + result.push(val as u8); + i += 4; + } else { + result.push(bytes[i]); + i += 1; + } + } else { + result.push(bytes[i]); + i += 1; + } + } + String::from_utf8(result).unwrap_or_else(|_| s.to_string()) +} + +/// Convert a fixed-size C character array from `utsname` into a Rust string +/// +/// `uname(2)` NUL-terminates each field, but the scan is bounded by the array length so +/// that a field which is not terminated cannot be read out of bounds. +pub(super) fn utsname_field(field: &[libc::c_char]) -> String { + let bytes: Vec = field + .iter() + .take_while(|&&c| c != 0) + .map(|&c| c as u8) + .collect(); + String::from_utf8_lossy(&bytes).to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + // --------------------------------------------------------------------- + // Device-name classification + // --------------------------------------------------------------------- + + #[test] + fn test_is_virtual_disk() { + // Virtual devices: an accepted prefix followed by nothing but digits, + // equivalent to node_exporter's `^(z?ram|loop|fd)\d+$`. + assert!(is_virtual_disk("ram0")); + assert!(is_virtual_disk("ram15")); + assert!(is_virtual_disk("zram0")); + assert!(is_virtual_disk("loop0")); + assert!(is_virtual_disk("loop10")); + assert!(is_virtual_disk("fd0")); + + // Plain prefix matching would wrongly exclude these: the suffix must be + // digits only, and a bare prefix with no index is not a device instance. + assert!(!is_virtual_disk("ramdisk0")); + assert!(!is_virtual_disk("loopback")); + assert!(!is_virtual_disk("ram")); + assert!(!is_virtual_disk("zram")); + assert!(!is_virtual_disk("loop")); + assert!(!is_virtual_disk("fd")); + assert!(!is_virtual_disk("loop0p1")); + + // Real disks are never excluded. + assert!(!is_virtual_disk("sda")); + assert!(!is_virtual_disk("nvme0n1")); + assert!(!is_virtual_disk("mmcblk0")); + } + + #[test] + fn test_is_partition() { + // Whole disks + assert!(!is_partition("sda")); + assert!(!is_partition("sdb")); + assert!(!is_partition("vda")); + assert!(!is_partition("xvda")); + assert!(!is_partition("nvme0n1")); + assert!(!is_partition("hda")); + + assert!(!is_partition("mmcblk0")); + assert!(!is_partition("mmcblk1")); + + // Multi-letter disk identifiers (e.g. 27th+ disk) + assert!(!is_partition("sdaa")); + assert!(!is_partition("sdab")); + + // Whole disks whose names end in a digit but delimit no partition + assert!(!is_partition("dm-0")); + assert!(!is_partition("md0")); + assert!(!is_partition("md127")); + assert!(!is_partition("sr0")); + assert!(!is_partition("loop0")); + + // Partitions + assert!(is_partition("sda1")); + assert!(is_partition("sda12")); + assert!(is_partition("sdaa1")); + assert!(is_partition("sdaa12")); + assert!(is_partition("vda1")); + assert!(is_partition("nvme0n1p1")); + assert!(is_partition("nvme0n1p12")); + assert!(is_partition("xvda1")); + assert!(is_partition("mmcblk0p1")); + assert!(is_partition("mmcblk0p12")); + assert!(is_partition("md127p1")); + assert!(is_partition("dasda1")); + } + + // --------------------------------------------------------------------- + // Path decoding and prefix matching + // --------------------------------------------------------------------- + + #[test] + fn test_decode_mount_path_no_escapes() { + assert_eq!(decode_mount_path("/mnt/data"), "/mnt/data"); + } + + #[test] + fn test_decode_mount_path_space() { + // \040 is space + assert_eq!(decode_mount_path("/mnt/my\\040drive"), "/mnt/my drive"); + } + + #[test] + fn test_decode_mount_path_multiple_escapes() { + // \040 space, \011 tab, \012 newline, \134 backslash + assert_eq!(decode_mount_path("/mnt/a\\040b\\134c"), "/mnt/a b\\c"); + } + + #[test] + fn test_decode_mount_path_tab_and_newline() { + assert_eq!(decode_mount_path("a\\011b\\012c"), "a\tb\nc"); + } + + #[test] + fn test_decode_mount_path_trailing_backslash() { + // Incomplete escape at end should be preserved as-is + assert_eq!(decode_mount_path("/mnt/foo\\04"), "/mnt/foo\\04"); + } + + #[test] + fn test_decode_mount_path_non_octal_after_backslash() { + // \9 is not a valid octal digit, should be preserved + assert_eq!(decode_mount_path("/mnt/foo\\9bar"), "/mnt/foo\\9bar"); + } + + #[test] + fn test_matches_path_prefix_exact() { + assert!(matches_path_prefix("/dev", "/dev")); + assert!(matches_path_prefix("/proc", "/proc")); + } + + #[test] + fn test_matches_path_prefix_subpath() { + assert!(matches_path_prefix("/dev/shm", "/dev")); + assert!(matches_path_prefix("/proc/sys/fs", "/proc")); + assert!(matches_path_prefix( + "/var/lib/docker/overlay2", + "/var/lib/docker" + )); + assert!(matches_path_prefix( + "/var/lib/containers/storage/overlay", + "/var/lib/containers" + )); + } + + #[test] + fn test_matches_path_prefix_no_false_positives() { + // Must not match when the prefix is only a substring, not a path boundary + assert!(!matches_path_prefix("/developer", "/dev")); + assert!(!matches_path_prefix("/devops", "/dev")); + assert!(!matches_path_prefix("/processing", "/proc")); + assert!(!matches_path_prefix( + "/var/lib/dockerfiles", + "/var/lib/docker" + )); + assert!(!matches_path_prefix( + "/var/lib/containers2", + "/var/lib/containers" + )); + } + + #[test] + fn test_matches_path_prefix_no_match() { + assert!(!matches_path_prefix("/mnt/data", "/dev")); + assert!(!matches_path_prefix("/home/user", "/proc")); + } + + #[test] + fn test_matches_path_prefix_root_and_empty_prefix() { + // The root prefix trims to the empty string, so it cannot be compared as a + // path component: it has to match every absolute path instead. + assert!(matches_path_prefix("/", "/")); + assert!(matches_path_prefix("/x", "/")); + assert!(matches_path_prefix("/var/lib/docker", "/")); + + // An empty prefix is not the root and must match nothing, otherwise a blank + // configured exclusion would silently drop every filesystem. + assert!(!matches_path_prefix("/x", "")); + assert!(!matches_path_prefix("", "")); + + // A relative path is not below the root prefix. + assert!(!matches_path_prefix("x", "/")); + } + + #[test] + fn test_matches_path_prefix_ignores_trailing_slash_on_prefix() { + // A trailing slash must behave exactly like no trailing slash, so a + // configured "/dev/" is not silently inert. + for (path, expected) in [ + ("/dev", true), + ("/dev/shm", true), + ("/developer", false), + ("/mnt/data", false), + ] { + assert_eq!( + matches_path_prefix(path, "/dev/"), + expected, + "prefix /dev/ against {}", + path + ); + assert_eq!( + matches_path_prefix(path, "/dev/"), + matches_path_prefix(path, "/dev"), + "trailing slash changed the result for {}", + path + ); + } + } + + // --------------------------------------------------------------------- + // utsname fields + // --------------------------------------------------------------------- + + /// Build a `utsname`-style field from raw bytes. + /// + /// `libc::c_char` is signed on most targets but unsigned on others (aarch64 Linux), + /// so the cast rather than a literal type keeps this portable. + fn utsname_bytes(bytes: &[u8]) -> Vec { + bytes.iter().map(|&b| b as libc::c_char).collect() + } + + #[test] + fn test_utsname_field_stops_at_nul() { + // uname(2) NUL-terminates each field and leaves the remainder of the array + // undefined; everything past the terminator must be ignored. + let field = utsname_bytes(b"Linux\0-leftover-garbage\0"); + assert_eq!(utsname_field(&field), "Linux"); + + // An empty field is an empty string, not a stray NUL character. + assert_eq!(utsname_field(&utsname_bytes(b"\0\0\0")), ""); + assert_eq!(utsname_field(&[]), ""); + } + + #[test] + fn test_utsname_field_without_terminator_truncates_at_array_length() { + // A field that fills the array with no room for the terminator must stop at + // the end of the slice rather than reading past it. + let field = utsname_bytes(b"6.1.0-no-nul-here"); + assert_eq!(utsname_field(&field), "6.1.0-no-nul-here"); + assert_eq!(utsname_field(&field).len(), field.len()); + } + + #[test] + fn test_utsname_field_converts_non_utf8_lossily() { + // The kernel does not promise UTF-8 (a hostname can hold arbitrary bytes), and + // a lossy conversion is preferable to dropping the whole label. + let field = utsname_bytes(&[b'a', 0xFF, b'b', 0]); + assert_eq!(utsname_field(&field), "a\u{FFFD}b"); + + // A truncated multi-byte sequence is replaced, not silently dropped. + assert_eq!(utsname_field(&utsname_bytes(&[0xC3, 0x28, 0])), "\u{FFFD}("); + } +} diff --git a/src/receivers/node_metrics/config.rs b/src/receivers/node_metrics/config.rs new file mode 100644 index 00000000..0abfd946 --- /dev/null +++ b/src/receivers/node_metrics/config.rs @@ -0,0 +1,323 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Configuration for the node metrics receiver + +use std::time::Duration; + +/// Default procfs mount point +pub const DEFAULT_PROCFS_PATH: &str = "/proc"; +/// Default sysfs mount point +pub const DEFAULT_SYSFS_PATH: &str = "/sys"; + +/// Available metric collectors +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Collector { + /// CPU metrics (seconds per mode: user, system, idle, iowait, etc.) + Cpu, + /// Load average metrics (node_load1, node_load5, node_load15) + Loadavg, + /// Memory metrics (all fields from /proc/meminfo) + Memory, + /// Network interface metrics (bytes/packets rx/tx per interface) + Network, + /// Filesystem metrics (size/free/avail space per mount) + Filesystem, + /// System information (uname: kernel, hostname) + Uname, + /// Kernel stat counters (forks, context switches, interrupts, procs running/blocked) from /proc/stat + Stat, + /// Process metrics (kernel limits: threads-max, pid_max) from /proc/sys/kernel/ + Processes, + /// Disk I/O statistics from /proc/diskstats + Diskstats, + /// Virtual memory statistics from /proc/vmstat + Vmstat, + /// Network statistics from /proc/net/netstat and /proc/net/snmp + Netstat, + /// Socket statistics from /proc/net/sockstat and /proc/net/sockstat6 + Sockstat, + /// File descriptor statistics from /proc/sys/fs/file-nr + Filefd, + /// CPU frequency from /sys/devices/system/cpu/cpu*/cpufreq/ + Cpufreq, + /// Thermal zone temperatures from /sys/class/thermal/thermal_zone*/ + ThermalZone, + /// NVMe device info from /sys/class/nvme/ + Nvme, + /// Hardware monitoring sensors from /sys/class/hwmon/ + Hwmon, + /// Custom metrics from Prometheus-format text files + Textfile, + /// System time (node_time_seconds, node_boot_time_seconds) + Time, +} + +/// Configuration for the node metrics receiver +/// +/// This struct cannot derive `Deserialize` because `regex::Regex` does not implement it. +/// Supporting file-based configuration (YAML/TOML) would mean adding a raw config struct +/// with `String` filter fields that validates into this one. +#[derive(Clone, Debug)] +pub struct NodeMetricsConfig { + /// Interval between metric collections + pub scrape_interval: Duration, + /// Which collectors to enable + pub collectors: Vec, + /// Optional regex pattern to include only matching metric names + pub metric_include_filter: Option, + /// Optional regex pattern to exclude matching metric names + pub metric_exclude_filter: Option, + /// Optional regex pattern to exclude matching filesystem mount points, + /// in addition to the built-in virtual and network filesystem exclusions + pub filesystem_mount_exclude: Option, + /// Path to procfs mount (default: /proc) + pub procfs_path: String, + /// Path to sysfs mount (default: /sys) + pub sysfs_path: String, + /// Prefix under which the host's root filesystem is reachable, or "/" when the + /// collector runs directly on the host. + /// + /// Needed by the filesystem collector: the mount *table* can be read from a + /// bind-mounted host procfs, but `statfs` still has to be called on a path that + /// resolves in this process's own mount namespace. + pub rootfs_path: String, + /// Directory containing Prometheus-format textfiles (*.prom), or a single .prom file + pub textfile_directory: Option, + /// Service name used in the OTLP Resource (default: "node_metrics") + pub service_name: String, +} + +impl Default for NodeMetricsConfig { + fn default() -> Self { + Self { + scrape_interval: Duration::from_secs(60), + collectors: vec![ + Collector::Cpu, + Collector::Loadavg, + Collector::Memory, + Collector::Network, + Collector::Filesystem, + Collector::Uname, + Collector::Stat, + Collector::Processes, + Collector::Diskstats, + Collector::Vmstat, + Collector::Netstat, + Collector::Sockstat, + Collector::Filefd, + Collector::Cpufreq, + Collector::ThermalZone, + Collector::Nvme, + Collector::Hwmon, + Collector::Time, + // Textfile not included by default - requires a directory to be set + ], + metric_include_filter: None, + metric_exclude_filter: None, + filesystem_mount_exclude: None, + procfs_path: DEFAULT_PROCFS_PATH.to_string(), + sysfs_path: DEFAULT_SYSFS_PATH.to_string(), + rootfs_path: "/".to_string(), + textfile_directory: None, + service_name: "node_metrics".to_string(), + } + } +} + +impl NodeMetricsConfig { + /// Create a new config with the specified scrape interval in seconds + pub fn new(scrape_interval_secs: u64) -> Self { + Self { + scrape_interval: Duration::from_secs(scrape_interval_secs), + ..Default::default() + } + } + + /// Set which collectors to enable + pub fn with_collectors(mut self, collectors: Vec) -> Self { + self.collectors = collectors; + self + } + + /// Set the metric include filter regex pattern + pub fn with_include_filter(mut self, pattern: &str) -> Result { + self.metric_include_filter = Some(regex::Regex::new(pattern)?); + Ok(self) + } + + /// Set the metric exclude filter regex pattern + pub fn with_exclude_filter(mut self, pattern: &str) -> Result { + self.metric_exclude_filter = Some(regex::Regex::new(pattern)?); + Ok(self) + } + + /// Set the filesystem mount point exclude regex pattern + pub fn with_filesystem_mount_exclude(mut self, pattern: &str) -> Result { + self.filesystem_mount_exclude = Some(regex::Regex::new(pattern)?); + Ok(self) + } + + /// Check if a metric name passes the configured filters + pub fn should_include_metric(&self, name: &str) -> bool { + // If include filter is set, metric must match it + if let Some(ref include) = self.metric_include_filter + && !include.is_match(name) + { + return false; + } + + // If exclude filter is set, metric must not match it + if let Some(ref exclude) = self.metric_exclude_filter + && exclude.is_match(name) + { + return false; + } + + true + } + + /// Validate the configuration and normalize collector list (deduplicate) + pub fn normalize_and_validate(&mut self) -> Result<(), String> { + if self.scrape_interval < Duration::from_secs(1) { + return Err(format!( + "Scrape interval must be at least 1 second, got {:?}", + self.scrape_interval + )); + } + + if self.collectors.is_empty() { + return Err("At least one collector must be enabled".to_string()); + } + + // Deduplicate collectors while preserving order + let mut seen = std::collections::HashSet::new(); + self.collectors.retain(|c| seen.insert(c.clone())); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The collector set enabled by default (textfile requires an explicit directory) + const DEFAULT_COLLECTORS: [Collector; 18] = [ + Collector::Cpu, + Collector::Loadavg, + Collector::Memory, + Collector::Network, + Collector::Filesystem, + Collector::Uname, + Collector::Stat, + Collector::Processes, + Collector::Diskstats, + Collector::Vmstat, + Collector::Netstat, + Collector::Sockstat, + Collector::Filefd, + Collector::Cpufreq, + Collector::ThermalZone, + Collector::Nvme, + Collector::Hwmon, + Collector::Time, + ]; + + #[test] + fn test_default_config() { + let config = NodeMetricsConfig::default(); + assert_eq!(config.scrape_interval, Duration::from_secs(60)); + // Compare the actual set, not just its size: a count assertion would still pass + // if one collector were swapped for another. + assert_eq!(config.collectors, DEFAULT_COLLECTORS.to_vec()); + assert!(config.metric_include_filter.is_none()); + assert!(config.metric_exclude_filter.is_none()); + assert!(!config.collectors.contains(&Collector::Textfile)); + } + + #[test] + fn test_normalize_deduplicates_collectors() { + let mut config = NodeMetricsConfig::default().with_collectors(vec![ + Collector::Cpu, + Collector::Memory, + Collector::Cpu, + ]); + config.normalize_and_validate().unwrap(); + assert_eq!(config.collectors, vec![Collector::Cpu, Collector::Memory]); + } + + #[test] + fn test_validate_rejects_sub_second_interval_and_reports_it() { + let mut config = NodeMetricsConfig { + scrape_interval: Duration::from_millis(500), + ..Default::default() + }; + let err = config.normalize_and_validate().unwrap_err(); + // The rejected value must be reported accurately, not truncated to "0s" + assert!( + err.contains("500ms"), + "error should report the offending duration, got: {}", + err + ); + } + + #[test] + fn test_validate_rejects_empty_collector_list() { + let mut config = NodeMetricsConfig::default().with_collectors(vec![]); + assert!(config.normalize_and_validate().is_err()); + } + + #[test] + fn test_exclude_filter_drops_matching_metrics() { + let config = NodeMetricsConfig::default() + .with_exclude_filter("^node_vmstat_") + .unwrap(); + assert!(!config.should_include_metric("node_vmstat_pgfault")); + assert!(config.should_include_metric("node_load1")); + } + + #[test] + fn test_include_filter_takes_precedence_over_unmatched() { + let config = NodeMetricsConfig::default() + .with_include_filter("^node_load") + .unwrap(); + assert!(config.should_include_metric("node_load1")); + assert!(!config.should_include_metric("node_cpu_seconds_total")); + } + + #[test] + fn test_config_with_collectors() { + let config = + NodeMetricsConfig::new(30).with_collectors(vec![Collector::Loadavg, Collector::Memory]); + assert_eq!(config.scrape_interval, Duration::from_secs(30)); + assert_eq!( + config.collectors, + vec![Collector::Loadavg, Collector::Memory] + ); + } + + #[test] + fn test_combined_filters() { + let config = NodeMetricsConfig::new(60) + .with_include_filter("node_.*") + .unwrap() + .with_exclude_filter(".*packets.*") + .unwrap(); + + assert!(config.should_include_metric("node_load1")); + assert!(config.should_include_metric("node_network_receive_bytes_total")); + assert!(!config.should_include_metric("node_network_receive_packets_total")); + assert!(!config.should_include_metric("other_metric")); + } + + #[test] + fn test_filesystem_mount_exclude() { + let config = NodeMetricsConfig::new(60) + .with_filesystem_mount_exclude("^/nfs/") + .unwrap(); + + let re = config.filesystem_mount_exclude.as_ref().unwrap(); + assert!(re.is_match("/nfs/share")); + assert!(!re.is_match("/mnt/data")); + } +} diff --git a/src/receivers/node_metrics/convert.rs b/src/receivers/node_metrics/convert.rs new file mode 100644 index 00000000..97f36ea0 --- /dev/null +++ b/src/receivers/node_metrics/convert.rs @@ -0,0 +1,684 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Convert collected system metrics to OTLP ResourceMetrics format + +use crate::receivers::node_metrics::collector::{CollectedMetric, MetricType}; +use gethostname::gethostname; +use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value}; +use opentelemetry_proto::tonic::metrics::v1::{ + AggregationTemporality, Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, + metric, number_data_point, +}; +use opentelemetry_proto::tonic::resource::v1::Resource; +use std::collections::{BTreeMap, HashSet}; +use std::time::{SystemTime, UNIX_EPOCH}; + +const SCOPE_NAME: &str = "node_metrics"; + +// Resource attributes +const SERVICE_NAME_KEY: &str = "service.name"; +const HOST_NAME_KEY: &str = "host.name"; +#[cfg(target_os = "linux")] +const OS_TYPE_KEY: &str = "os.type"; +#[cfg(target_os = "linux")] +const OS_TYPE_VALUE: &str = "linux"; + +/// Convert a batch of collected metrics to OTLP ResourceMetrics +/// +/// `boot_time_secs` is the system boot time in seconds since epoch, used as +/// the start time for cumulative counters. +/// +/// `service_name` is the value set on the `service.name` resource attribute. +pub fn convert_to_otlp_metrics( + metrics: Vec, + boot_time_secs: u64, + service_name: &str, +) -> ResourceMetrics { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + let now_ns = now.as_secs() * 1_000_000_000 + now.subsec_nanos() as u64; + + let boot_time_ns = boot_time_secs.saturating_mul(1_000_000_000); + + // Group metrics by name *and* type so each OTLP Metric contains all its data points. + // Keying on the type as well keeps a gauge and a counter that happen to share a name + // (reachable via textfiles, whose `# TYPE` scope is per file) as two separate + // metrics, rather than silently stamping one group's points with the other's + // temporality and start time. + // BTreeMap gives deterministic (alphabetical) ordering across scrapes, + // which simplifies debugging and diffing. + let mut grouped: BTreeMap<(String, MetricType), Vec> = BTreeMap::new(); + for mut m in metrics { + // The name moves into the key; the grouped metrics keep an empty `name` field, + // which nothing below reads (only labels, value, unit and description are used). + let key = (std::mem::take(&mut m.name), m.metric_type); + grouped.entry(key).or_default().push(m); + } + + // A name that appears with two types stays two OTLP metrics, which a backend keying + // on resource+scope+name still treats as a conflict. Surface it rather than letting + // it be resolved arbitrarily downstream. + let mut previous: Option<&String> = None; + for (name, _) in grouped.keys() { + if previous == Some(name) { + // Debug, not warn: a conflicting `# TYPE` persists until the file is fixed, + // so warning would log this on every scrape forever. The same reasoning the + // textfile collector applies to its own repeating failures. + tracing::debug!( + "Metric {} is reported as both a gauge and a counter; check for a \ + conflicting `# TYPE` in the textfile directory", + name + ); + } + previous = Some(name); + } + + let otlp_metrics: Vec = grouped + .into_iter() + .map(|((name, metric_type), group)| { + convert_metric_group(name, metric_type, group, now_ns, boot_time_ns) + }) + .collect(); + + ResourceMetrics { + resource: Some(build_resource(service_name)), + scope_metrics: vec![ScopeMetrics { + scope: Some( + opentelemetry_proto::tonic::common::v1::InstrumentationScope { + name: SCOPE_NAME.to_string(), + version: String::new(), + attributes: vec![], + dropped_attributes_count: 0, + }, + ), + metrics: otlp_metrics, + schema_url: String::new(), + }], + schema_url: String::new(), + } +} + +/// Count the data points in a converted batch +/// +/// Every variant is matched explicitly: a silent `_ => 0` would under-count without any +/// compile error if a new metric shape were ever emitted here. +pub(crate) fn count_data_points(resource_metrics: &ResourceMetrics) -> usize { + resource_metrics + .scope_metrics + .iter() + .flat_map(|sm| sm.metrics.iter()) + .map(|m| match &m.data { + Some(metric::Data::Gauge(g)) => g.data_points.len(), + Some(metric::Data::Sum(s)) => s.data_points.len(), + Some(metric::Data::Histogram(h)) => h.data_points.len(), + Some(metric::Data::ExponentialHistogram(h)) => h.data_points.len(), + Some(metric::Data::Summary(s)) => s.data_points.len(), + None => 0, + }) + .sum() +} + +/// Build the OTLP Resource identifying the host these metrics describe +/// +/// `host.name` matters here: without it, the same series from different hosts share an +/// identical identity at the backend. Mirrors the kmsg receiver's resource attributes. +fn build_resource(service_name: &str) -> Resource { + let mut attributes = vec![string_attribute(SERVICE_NAME_KEY, service_name.to_string())]; + + if let Ok(hostname) = gethostname().into_string() { + attributes.push(string_attribute(HOST_NAME_KEY, hostname)); + } + + #[cfg(target_os = "linux")] + attributes.push(string_attribute(OS_TYPE_KEY, OS_TYPE_VALUE.to_string())); + + Resource { + attributes, + dropped_attributes_count: 0, + entity_refs: vec![], + } +} + +/// Build a string-valued OTLP attribute +fn string_attribute(key: &str, value: String) -> KeyValue { + KeyValue { + key: key.to_string(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(value)), + }), + + key_strindex: 0, + } +} + +/// Convert a group of collected metrics (same name and type) to a single OTLP Metric +fn convert_metric_group( + name: String, + metric_type: MetricType, + group: Vec, + time_unix_nano: u64, + boot_time_nano: u64, +) -> Metric { + // Take the remaining metadata from the first metric in the group + let (description, unit) = match group.first() { + Some(first) => ( + first.description.clone().unwrap_or_default(), + first.unit.clone().unwrap_or_default(), + ), + None => (String::new(), String::new()), + }; + + // For counters, start_time is the boot time (when counting began). + // For gauges, start_time is not applicable (left unset). + let start_time = match metric_type { + MetricType::Counter => boot_time_nano, + MetricType::Gauge => 0, + }; + + // Duplicate attribute sets within one metric violate the OTLP single-writer + // principle and are rejected (or arbitrarily resolved) by backends. They are + // reachable via textfiles and duplicated mount entries, so drop repeats here. + let mut seen: HashSet> = HashSet::with_capacity(group.len()); + let mut dropped = 0usize; + let mut data_points: Vec = Vec::with_capacity(group.len()); + + for mut m in group { + // Compare label *sets*: `{a="1",b="2"}` and `{b="2",a="1"}` are the same + // OTLP identity, and textfile labels arrive in whatever order the file used. + // Sorting in place also makes the emitted attribute order stable across scrapes. + m.labels.sort_unstable(); + if !seen.insert(m.labels.clone()) { + dropped += 1; + continue; + } + + let attributes: Vec = m + .labels + .into_iter() + .map(|(k, v)| KeyValue { + key: k, + value: Some(AnyValue { + value: Some(any_value::Value::StringValue(v)), + }), + + key_strindex: 0, + }) + .collect(); + + data_points.push(NumberDataPoint { + attributes, + start_time_unix_nano: start_time, + time_unix_nano, + exemplars: vec![], + flags: 0, + value: Some(number_data_point::Value::AsDouble(m.value)), + }); + } + + if dropped > 0 { + // Debug for the same reason: duplicated mounts or a bad `.prom` file do not + // self-heal, so this would repeat every scrape. + tracing::debug!( + "Dropped {} duplicate data point(s) for metric {}", + dropped, + name + ); + } + + let data = match metric_type { + MetricType::Gauge => { + opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(Gauge { data_points }) + } + MetricType::Counter => opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(Sum { + data_points, + aggregation_temporality: AggregationTemporality::Cumulative as i32, + is_monotonic: true, + }), + }; + + Metric { + name, + description, + unit, + metadata: vec![], + data: Some(data), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_test_metric( + name: &str, + value: f64, + metric_type: MetricType, + labels: Vec<(String, String)>, + ) -> CollectedMetric { + CollectedMetric { + name: name.to_string(), + value, + labels, + metric_type, + unit: Some("By".to_string()), + description: Some("Test metric".to_string()), + } + } + + #[test] + fn test_convert_gauge_metric() { + let metric = make_test_metric("node_load1", 1.5, MetricType::Gauge, vec![]); + + let resource_metrics = convert_to_otlp_metrics(vec![metric], 1234567890, "node_metrics"); + + assert!(resource_metrics.resource.is_some()); + assert_eq!(resource_metrics.scope_metrics.len(), 1); + assert_eq!(resource_metrics.scope_metrics[0].metrics.len(), 1); + + let otlp_metric = &resource_metrics.scope_metrics[0].metrics[0]; + assert_eq!(otlp_metric.name, "node_load1"); + + match &otlp_metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(gauge)) => { + assert_eq!(gauge.data_points.len(), 1); + match gauge.data_points[0].value { + Some(number_data_point::Value::AsDouble(v)) => { + assert!((v - 1.5).abs() < f64::EPSILON); + } + _ => panic!("Expected double value"), + } + } + _ => panic!("Expected Gauge data"), + } + } + + #[test] + fn test_convert_counter_metric() { + let metric = make_test_metric( + "node_network_receive_bytes_total", + 1000.0, + MetricType::Counter, + vec![], + ); + + let resource_metrics = convert_to_otlp_metrics(vec![metric], 1234567890, "node_metrics"); + + let otlp_metric = &resource_metrics.scope_metrics[0].metrics[0]; + assert_eq!(otlp_metric.name, "node_network_receive_bytes_total"); + + match &otlp_metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(sum)) => { + assert!(sum.is_monotonic); + assert_eq!( + sum.aggregation_temporality, + AggregationTemporality::Cumulative as i32 + ); + } + _ => panic!("Expected Sum data"), + } + } + + #[test] + fn test_convert_metric_with_labels() { + let labels = vec![("device".to_string(), "eth0".to_string())]; + + let metric = make_test_metric( + "node_network_receive_bytes_total", + 1000.0, + MetricType::Counter, + labels, + ); + + let resource_metrics = convert_to_otlp_metrics(vec![metric], 1234567890, "node_metrics"); + + let otlp_metric = &resource_metrics.scope_metrics[0].metrics[0]; + + match &otlp_metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(sum)) => { + assert_eq!(sum.data_points[0].attributes.len(), 1); + assert_eq!(sum.data_points[0].attributes[0].key, "device"); + // The value has to survive the conversion too: a label whose key is + // carried but whose value is dropped would still pass a key-only check. + assert_eq!( + attribute_pairs(&sum.data_points[0].attributes), + vec![("device", "eth0")] + ); + } + _ => panic!("Expected Sum data"), + } + } + + #[test] + fn test_convert_multiple_metrics_same_name_grouped() { + // Multiple data points for the same metric name should be grouped + let metrics = vec![ + make_test_metric( + "node_cpu_seconds_total", + 10.0, + MetricType::Counter, + vec![ + ("cpu".to_string(), "0".to_string()), + ("mode".to_string(), "user".to_string()), + ], + ), + make_test_metric( + "node_cpu_seconds_total", + 20.0, + MetricType::Counter, + vec![ + ("cpu".to_string(), "0".to_string()), + ("mode".to_string(), "system".to_string()), + ], + ), + ]; + + let resource_metrics = convert_to_otlp_metrics(metrics, 1234567890, "node_metrics"); + + // Should produce a single Metric with 2 data points + assert_eq!(resource_metrics.scope_metrics[0].metrics.len(), 1); + match &resource_metrics.scope_metrics[0].metrics[0].data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(sum)) => { + assert_eq!(sum.data_points.len(), 2); + // Which two points survive matters as much as how many: grouping that + // kept one point twice, or swapped a value onto the wrong attribute + // set, would still leave the count at 2. + assert_eq!( + point_summaries(&sum.data_points), + vec![ + (vec![("cpu", "0"), ("mode", "user")], 10.0), + (vec![("cpu", "0"), ("mode", "system")], 20.0), + ] + ); + } + _ => panic!("Expected Sum data"), + } + } + + #[test] + fn test_resource_has_service_name() { + let metric = make_test_metric("node_load1", 1.0, MetricType::Gauge, vec![]); + + let resource_metrics = + convert_to_otlp_metrics(vec![metric], 1234567890, "my_custom_service"); + + let resource = resource_metrics.resource.unwrap(); + + assert_eq!( + attribute_value(&resource, "service.name").as_deref(), + Some("my_custom_service") + ); + + // host.name must be present so series from different hosts stay distinguishable. + // The value itself is host-dependent and may be empty in a minimal container. + assert!( + resource.attributes.iter().any(|a| a.key == "host.name"), + "expected a host.name resource attribute, got {:?}", + resource.attributes + ); + + #[cfg(target_os = "linux")] + assert_eq!( + attribute_value(&resource, "os.type").as_deref(), + Some("linux") + ); + } + + /// The `(key, value)` pairs of a data point's attributes, in emitted order + fn attribute_pairs(attributes: &[KeyValue]) -> Vec<(&str, &str)> { + attributes + .iter() + .map(|kv| { + let value = match &kv.value { + Some(AnyValue { + value: Some(any_value::Value::StringValue(v)), + }) => v.as_str(), + other => panic!( + "expected a string attribute for {}, got {:?}", + kv.key, other + ), + }; + (kv.key.as_str(), value) + }) + .collect() + } + + /// The `(attributes, value)` of every data point, in emitted order + fn point_summaries(points: &[NumberDataPoint]) -> Vec<(Vec<(&str, &str)>, f64)> { + points + .iter() + .map(|p| { + let value = match p.value { + Some(number_data_point::Value::AsDouble(v)) => v, + other => panic!("expected a double data point value, got {:?}", other), + }; + (attribute_pairs(&p.attributes), value) + }) + .collect() + } + + /// The data points of the single metric in a converted batch + fn only_data_points(resource_metrics: &ResourceMetrics) -> &[NumberDataPoint] { + let metrics = &resource_metrics.scope_metrics[0].metrics; + assert_eq!(metrics.len(), 1, "expected exactly one metric"); + match &metrics[0].data { + Some(metric::Data::Gauge(gauge)) => &gauge.data_points, + Some(metric::Data::Sum(sum)) => &sum.data_points, + other => panic!("expected gauge or sum data, got {:?}", other), + } + } + + /// Look up a string-valued resource attribute by key + fn attribute_value(resource: &Resource, key: &str) -> Option { + resource + .attributes + .iter() + .find(|a| a.key == key) + .and_then(|a| match &a.value { + Some(AnyValue { + value: Some(any_value::Value::StringValue(v)), + }) => Some(v.clone()), + _ => None, + }) + } + + #[test] + fn test_same_name_gauge_and_counter_stay_separate() { + // Reachable through textfiles, whose `# TYPE` scope is per file: the two must not + // be merged into one metric with a single temporality. + let gauge = make_test_metric("collision", 1.0, MetricType::Gauge, vec![]); + let counter = make_test_metric("collision", 2.0, MetricType::Counter, vec![]); + + let resource_metrics = convert_to_otlp_metrics(vec![gauge, counter], 1000, "node_metrics"); + let metrics = &resource_metrics.scope_metrics[0].metrics; + + assert_eq!(metrics.len(), 2, "expected separate gauge and sum metrics"); + assert!(metrics.iter().all(|m| m.name == "collision")); + + let has_gauge = metrics.iter().any(|m| { + matches!( + m.data, + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(_)) + ) + }); + let sum = metrics.iter().find_map(|m| match &m.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(s)) => Some(s), + _ => None, + }); + + assert!(has_gauge, "gauge variant missing"); + let sum = sum.expect("sum variant missing"); + // The counter's points keep the boot-time start, the gauge's stay unset + assert_eq!( + sum.data_points[0].start_time_unix_nano, + 1000 * 1_000_000_000 + ); + } + + #[test] + fn test_duplicate_attribute_sets_are_dropped() { + // Two samples with identical label sets would violate the OTLP single-writer + // principle and can be rejected wholesale by backends. + let labels = vec![("device".to_string(), "sda".to_string())]; + let first = make_test_metric("node_disk_io_now", 1.0, MetricType::Gauge, labels.clone()); + let duplicate = make_test_metric("node_disk_io_now", 2.0, MetricType::Gauge, labels); + let other = make_test_metric( + "node_disk_io_now", + 3.0, + MetricType::Gauge, + vec![("device".to_string(), "sdb".to_string())], + ); + + let resource_metrics = + convert_to_otlp_metrics(vec![first, duplicate, other], 1000, "node_metrics"); + let metric = &resource_metrics.scope_metrics[0].metrics[0]; + + match &metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(gauge)) => { + assert_eq!(gauge.data_points.len(), 2); + // First occurrence wins, and the point that is *not* a duplicate keeps + // its own value: dropping `sdb` instead would also leave two points. + assert_eq!( + point_summaries(&gauge.data_points), + vec![ + (vec![("device", "sda")], 1.0), + (vec![("device", "sdb")], 3.0), + ] + ); + } + _ => panic!("Expected Gauge data"), + } + } + + #[test] + fn test_duplicate_attribute_sets_are_detected_regardless_of_label_order() { + // The same OTLP identity written with its labels in two different orders, as + // textfile lines and repeated mount entries can produce. Comparing the label + // vectors as given would treat these as two distinct series, so both would be + // emitted and the backend would reject or arbitrarily resolve them. + let unsorted = make_test_metric( + "node_disk_io_now", + 1.0, + MetricType::Gauge, + vec![ + ("b".to_string(), "2".to_string()), + ("a".to_string(), "1".to_string()), + ], + ); + let sorted = make_test_metric( + "node_disk_io_now", + 2.0, + MetricType::Gauge, + vec![ + ("a".to_string(), "1".to_string()), + ("b".to_string(), "2".to_string()), + ], + ); + + let resource_metrics = + convert_to_otlp_metrics(vec![unsorted, sorted], 1000, "node_metrics"); + let points = only_data_points(&resource_metrics); + + // One point survives, and its attributes are emitted in sorted order: the + // in-place sort that makes the comparison order-insensitive also normalises + // what is exported, so the attribute order is stable across scrapes. + assert_eq!( + point_summaries(points), + vec![(vec![("a", "1"), ("b", "2")], 1.0)] + ); + } + + #[test] + fn test_count_data_points_counts_exported_points_not_collected_ones() { + // A duplicate label set is dropped during conversion, so the collected count + // over-reports what was actually exported. + let labels = vec![("device".to_string(), "sda".to_string())]; + let collected = vec![ + make_test_metric("node_disk_io_now", 1.0, MetricType::Gauge, labels.clone()), + make_test_metric("node_disk_io_now", 2.0, MetricType::Gauge, labels), + make_test_metric( + "node_disk_io_now", + 3.0, + MetricType::Gauge, + vec![("device".to_string(), "sdb".to_string())], + ), + // A counter as well, so the Sum arm of the count is exercised too. + make_test_metric( + "node_disk_reads_completed_total", + 4.0, + MetricType::Counter, + vec![("device".to_string(), "sda".to_string())], + ), + ]; + let collected_count = collected.len(); + + let resource_metrics = convert_to_otlp_metrics(collected, 1000, "node_metrics"); + + // Two gauge points (the duplicate dropped) plus one counter point. + assert_eq!(count_data_points(&resource_metrics), 3); + assert_ne!( + count_data_points(&resource_metrics), + collected_count, + "the point count must reflect the conversion's drops" + ); + + // Cross-check against the emitted metrics rather than a hard-coded number. + let emitted: usize = resource_metrics.scope_metrics[0] + .metrics + .iter() + .map(|m| match &m.data { + Some(metric::Data::Gauge(g)) => g.data_points.len(), + Some(metric::Data::Sum(s)) => s.data_points.len(), + other => panic!("unexpected metric data {:?}", other), + }) + .sum(); + assert_eq!(count_data_points(&resource_metrics), emitted); + } + + #[test] + fn test_count_data_points_of_an_empty_batch_is_zero() { + let resource_metrics = convert_to_otlp_metrics(vec![], 1000, "node_metrics"); + assert_eq!(count_data_points(&resource_metrics), 0); + } + + #[test] + fn test_gauge_start_time_is_unset_and_metadata_propagates() { + let metric = make_test_metric("node_load1", 1.5, MetricType::Gauge, vec![]); + + let resource_metrics = convert_to_otlp_metrics(vec![metric], 1234567890, "node_metrics"); + let otlp_metric = &resource_metrics.scope_metrics[0].metrics[0]; + + // unit/description come from the collected metric + assert_eq!(otlp_metric.unit, "By"); + assert_eq!(otlp_metric.description, "Test metric"); + + let scope = resource_metrics.scope_metrics[0].scope.as_ref().unwrap(); + assert_eq!(scope.name, "node_metrics"); + + match &otlp_metric.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(gauge)) => { + assert_eq!(gauge.data_points[0].start_time_unix_nano, 0); + assert!(gauge.data_points[0].time_unix_nano > 0); + } + _ => panic!("Expected Gauge data"), + } + } + + #[test] + fn test_metrics_are_grouped_alphabetically() { + let metrics = vec![ + make_test_metric("node_c", 1.0, MetricType::Gauge, vec![]), + make_test_metric("node_a", 2.0, MetricType::Gauge, vec![]), + make_test_metric("node_b", 3.0, MetricType::Gauge, vec![]), + ]; + + let resource_metrics = convert_to_otlp_metrics(metrics, 1000, "node_metrics"); + let names: Vec<&str> = resource_metrics.scope_metrics[0] + .metrics + .iter() + .map(|m| m.name.as_str()) + .collect(); + + assert_eq!(names, vec!["node_a", "node_b", "node_c"]); + } +} diff --git a/src/receivers/node_metrics/mod.rs b/src/receivers/node_metrics/mod.rs new file mode 100644 index 00000000..ba76f7d1 --- /dev/null +++ b/src/receivers/node_metrics/mod.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Node metrics receiver +//! +//! Collects system metrics (CPU, memory, load, network, disk, filesystem, etc.) and +//! converts them to OpenTelemetry metrics. + +pub mod collector; +pub mod config; +pub mod convert; +pub mod receiver; + +pub use config::NodeMetricsConfig; +pub use receiver::NodeMetricsReceiver; diff --git a/src/receivers/node_metrics/receiver.rs b/src/receivers/node_metrics/receiver.rs new file mode 100644 index 00000000..feb5c58b --- /dev/null +++ b/src/receivers/node_metrics/receiver.rs @@ -0,0 +1,896 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Node metrics receiver implementation +//! +//! Periodically collects system metrics and sends them to the pipeline. + +use crate::receivers::get_meter; +use crate::receivers::node_metrics::collector::{CollectedMetric, SystemCollector}; +use crate::receivers::node_metrics::config::NodeMetricsConfig; +use crate::receivers::node_metrics::convert::{convert_to_otlp_metrics, count_data_points}; +use crate::receivers::otlp_output::OTLPOutput; +use crate::topology::payload; +use opentelemetry::KeyValue; +use opentelemetry::metrics::Counter; +use opentelemetry_proto::tonic::metrics::v1::ResourceMetrics; +use std::sync::Arc; +use tokio::task::{JoinHandle, JoinSet}; +use tokio_util::sync::CancellationToken; +use tower::BoxError; +use tracing::{debug, error, info, warn}; + +/// Node metrics receiver that periodically scrapes system metrics +pub struct NodeMetricsReceiver { + config: NodeMetricsConfig, + metrics_output: Option>>, +} + +impl NodeMetricsReceiver { + /// Create a new node metrics receiver + /// + /// The configuration is validated here, as the sibling receivers do, so a config built + /// programmatically rather than from the CLI cannot bypass the checks. + pub fn new( + mut config: NodeMetricsConfig, + metrics_output: Option>>, + ) -> Result { + config.normalize_and_validate()?; + Ok(Self { + config, + metrics_output, + }) + } + + /// Start the receiver, spawning a task that periodically collects metrics + pub fn start( + self, + task_set: &mut JoinSet>, + receivers_cancel: &CancellationToken, + ) -> Result<(), BoxError> { + let cancel = receivers_cancel.clone(); + let config = self.config; + let metrics_output = self.metrics_output; + + info!( + scrape_interval = ?config.scrape_interval, + collectors = ?config.collectors, + "Node metrics receiver starting" + ); + + #[cfg(not(target_os = "linux"))] + warn!( + "Node metrics receiver is designed for Linux. \ + Most collectors read from /proc and /sys which may not be available on this platform." + ); + + task_set.spawn(async move { + let result = run_scrape_loop(config, metrics_output, cancel).await; + if let Err(ref e) = result { + error!("Node metrics receiver error: {}", e); + } + result + }); + + Ok(()) + } +} + +/// How long a final batch may take to hand off once cancellation has been requested +const SHUTDOWN_SEND_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(500); + +/// Lower bound on how long a scrape may take before it is considered wedged +/// +/// The timeout is the scrape interval or this, whichever is longer: a short interval should +/// not turn a merely slow host into a receiver that never reports at all. +const MIN_SCRAPE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// How long to wait for the pipeline to accept a batch before dropping it +/// +/// An upper bound, further capped by the scrape interval at each use site, so a batch is +/// never held past the next tick and a long interval does not buy a long stall. +const SEND_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + +/// Metrics counters for the receiver +struct ReceiverMetrics { + accepted: Counter, + refused: Counter, + scrape_failures: Counter, + empty_scrapes: Counter, + tags: [KeyValue; 1], +} + +impl ReceiverMetrics { + fn new() -> Self { + Self { + // Use the same accepted/refused vocabulary as the other receivers, so node + // metrics show up on the existing internal-telemetry dashboards. + // The description is part of the instrument identity, so it must match the + // other receivers declaring these names or the SDK reports a duplicate + // metric stream and exports two conflicting series. + accepted: get_meter() + .u64_counter("rotel_receiver_accepted_metric_points") + .with_description( + "Number of metric points successfully ingested and pushed into the pipeline.", + ) + .with_unit("metric_points") + .build(), + refused: get_meter() + .u64_counter("rotel_receiver_refused_metric_points") + .with_description( + "Number of metric points that could not be pushed into the pipeline.", + ) + .with_unit("metric_points") + .build(), + scrape_failures: get_meter() + .u64_counter("rotel_receiver_scrape_failures") + .with_description("Number of failed scrape attempts.") + .with_unit("scrapes") + .build(), + empty_scrapes: get_meter() + .u64_counter("rotel_receiver_empty_scrapes") + .with_description( + "Number of scrapes that collected no metrics at all, which usually means a \ + misconfigured procfs/sysfs path or an over-broad metric filter.", + ) + .with_unit("scrapes") + .build(), + tags: [KeyValue::new("receiver", "node_metrics")], + } + } + + fn add_accepted(&self, count: u64) { + if count > 0 { + self.accepted.add(count, &self.tags); + } + } + + fn add_refused(&self, count: u64) { + if count > 0 { + self.refused.add(count, &self.tags); + } + } + + fn add_scrape_failure(&self) { + self.scrape_failures.add(1, &self.tags); + } + + fn add_empty_scrape(&self) { + self.empty_scrapes.add(1, &self.tags); + } +} + +/// Decide whether a previously timed-out scrape is still occupying a blocking thread. +/// +/// Returns `true` when the earlier scrape has not finished, in which case this cycle must +/// be skipped: tokio cannot abort a blocking task, so starting another one would leak a +/// thread from the pool shared with the exporters every interval. A finished task is +/// reaped (surfacing a panic) and cleared, allowing scraping to resume. +async fn scrape_still_running(outstanding: &mut Option>>) -> bool { + let Some(handle) = outstanding else { + return false; + }; + + if !handle.is_finished() { + return true; + } + + if let Err(e) = handle.await { + warn!("Previously timed-out scrape task failed: {}", e); + } + *outstanding = None; + false +} + +/// Main scrape loop +async fn run_scrape_loop( + config: NodeMetricsConfig, + metrics_output: Option>>, + cancel: CancellationToken, +) -> Result<(), BoxError> { + let Some(output) = metrics_output else { + // Returning here would complete this receiver task, which the agent treats as an + // unexpected early exit and responds to by shutting everything down. Idle until + // cancellation instead. + warn!( + "No metrics output configured; node metrics receiver will not scrape. \ + Configure an exporter that accepts metrics." + ); + cancel.cancelled().await; + return Ok(()); + }; + + let receiver_metrics = ReceiverMetrics::new(); + let config = Arc::new(config); + + // Constructing the collector reads /proc/stat, so keep it off the async worker too. + let collector = { + let config = Arc::clone(&config); + Arc::new( + tokio::task::spawn_blocking(move || { + SystemCollector::new(&config.procfs_path, &config.sysfs_path) + }) + .await + .map_err(|e| format!("failed to initialize node metrics collector: {}", e))?, + ) + }; + + debug!( + "Node metrics scrape loop started, interval: {:?}, boot_time: {}", + config.scrape_interval, + collector.boot_time() + ); + + // Use interval_at to start immediately, then at regular intervals. + // Skip (not Delay or Burst) keeps ticks aligned to the original schedule after a slow + // scrape, instead of shifting the whole series forward or firing catch-up ticks. + let start = tokio::time::Instant::now(); + let mut interval = tokio::time::interval_at(start, config.scrape_interval); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + // A scrape that timed out keeps running: tokio cannot abort a blocking task once it + // has started. Hold on to it so the loop does not start a fresh scrape (and leak + // another blocking thread) every interval while the previous one is wedged — that + // would eventually exhaust the blocking pool shared with the exporters. + let mut outstanding: Option>> = None; + + loop { + tokio::select! { + biased; + + _ = cancel.cancelled() => { + info!("Node metrics receiver cancelled"); + break; + } + + _ = interval.tick() => {} + } + + if scrape_still_running(&mut outstanding).await { + warn!( + "Previous scrape is still running or queued, skipping this cycle. \ + If this persists, a filesystem or sysfs read is likely unresponsive; \ + a filesystem can be excluded with \ + --node-metrics-receiver-filesystem-mount-exclude." + ); + receiver_metrics.add_scrape_failure(); + continue; + } + + debug!("Scraping node metrics"); + + // Collect metrics on a blocking thread to avoid stalling the async runtime + // (statfs and /proc reads can block). The timeout bounds the loop, not the + // blocking task itself, and cancellation is honoured while waiting. + // Never shorter than MIN_SCRAPE_TIMEOUT: with the 1s interval the CLI permits, a + // scrape that legitimately takes longer would time out on every cycle and the + // receiver would emit nothing at all rather than merely lagging. + let scrape_timeout = config.scrape_interval.max(MIN_SCRAPE_TIMEOUT); + let mut handle = { + let collector = Arc::clone(&collector); + let config = Arc::clone(&config); + tokio::task::spawn_blocking(move || collector.collect(&config)) + }; + + let collected = tokio::select! { + biased; + + _ = cancel.cancelled() => { + info!("Node metrics receiver cancelled during scrape"); + break; + } + + res = tokio::time::timeout(scrape_timeout, &mut handle) => match res { + Ok(Ok(collected)) => collected, + Ok(Err(e)) => { + // Do not return Err: the agent propagates a receiver error *before* + // cancelling receivers and flushing exporters, so one panicking + // collector would discard unrelated pipelines' buffered telemetry. + // Treat it like the abandoned-task case and keep scraping. + warn!("Node metrics scrape task failed: {}", e); + receiver_metrics.add_scrape_failure(); + continue; + } + Err(_) => { + warn!("Scrape timed out after {:?}, skipping this cycle", scrape_timeout); + receiver_metrics.add_scrape_failure(); + // Keep the handle so the next tick can tell whether the blocking task + // is still wedged, instead of spawning another one on top of it. + outstanding = Some(handle); + continue; + } + } + }; + + let metric_count = collected.len(); + + if metric_count == 0 { + // Not debug-only: an empty scrape every interval means a wrong procfs/sysfs + // path or an over-broad exclude filter, which is otherwise invisible. + // Counted as well as logged: a permanently empty scrape is otherwise + // indistinguishable from a receiver that was never enabled. + warn!( + "No metrics collected; check the configured procfs/sysfs paths and metric filters" + ); + receiver_metrics.add_empty_scrape(); + continue; + } + + // Convert to OTLP format. Counters carry the *latched* boot time, so that a wall + // clock step does not move their start time and read as a counter reset. + let resource_metrics = convert_to_otlp_metrics( + collected, + collector.counter_start_time(), + &config.service_name, + ); + + // Count the points actually exported, not the ones collected: the conversion + // drops duplicate label sets. + let point_count = count_data_points(&resource_metrics); + + // Send to an output channel + let message = payload::Message::new(None, vec![resource_metrics], None); + + // Hold the send future so cancellation can give the in-flight send a bounded + // window to finish rather than discarding the batch outright. + let mut send_fut = std::pin::pin!(output.send_async(message)); + + tokio::select! { + biased; + + _ = cancel.cancelled() => { + // The pipeline is still draining at this point, so let the send finish if + // it can. The window stays well inside the agent's receiver drain budget. + match tokio::time::timeout(SHUTDOWN_SEND_TIMEOUT, send_fut.as_mut()).await { + Ok(Ok(())) => { + debug!("Sent {} metric points during shutdown", point_count); + receiver_metrics.add_accepted(point_count as u64); + } + Ok(Err(e)) => { + info!( + "Metrics pipeline already closed during shutdown, dropping {} \ + metric points: {}", + point_count, e + ); + receiver_metrics.add_refused(point_count as u64); + } + Err(_) => { + info!( + "Metrics pipeline did not accept {} metric points within {:?} of \ + shutdown, dropping them", + point_count, SHUTDOWN_SEND_TIMEOUT + ); + receiver_metrics.add_refused(point_count as u64); + } + } + break; + } + + // Bound the send: a full channel (slow or retrying exporter) would otherwise + // park here indefinitely, silently stopping all scraping. + res = tokio::time::timeout( + SEND_TIMEOUT.min(config.scrape_interval), + send_fut.as_mut(), + ) => match res { + Ok(Ok(())) => { + debug!("Sent {} metric points to pipeline", point_count); + receiver_metrics.add_accepted(point_count as u64); + } + Ok(Err(e)) => { + // The only send error is a disconnected pipeline. During shutdown that + // is expected, so do not report it as a receiver failure; otherwise + // retrying forever would hide it from the agent. + receiver_metrics.add_refused(point_count as u64); + if cancel.is_cancelled() { + info!("Metrics pipeline closed during shutdown: {}", e); + break; + } + error!("Failed to send metrics to pipeline: {}", e); + return Err(format!("node metrics pipeline disconnected: {}", e).into()); + } + Err(_) => { + // Drop this batch rather than stalling: the next scrape re-reads the + // current values, so a dropped batch is recoverable. + warn!( + "Metrics pipeline did not accept {} metric points within {:?}, \ + abandoning this batch (it may still be delivered). The metrics \ + exporter is likely backed up.", + point_count, + SEND_TIMEOUT.min(config.scrape_interval) + ); + receiver_metrics.add_refused(point_count as u64); + } + } + } + } + + info!("Node metrics receiver stopped"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bounded_channel::bounded; + use std::time::Duration; + use tracing_test::traced_test; + + use crate::receivers::node_metrics::config::Collector; + + /// Count captured log lines containing `needle`. + /// + /// `logs_assert` is the only accessor `#[traced_test]` provides, so the count is + /// captured out of the closure. + macro_rules! logs_with { + ($needle:expr) => {{ + let count = std::cell::Cell::new(0usize); + logs_assert(|lines: &[&str]| { + count.set(lines.iter().filter(|line| line.contains($needle)).count()); + Ok(()) + }); + count.get() + }}; + } + + /// Drive the loop until it has logged the start of a scrape, then give it a short grace + /// period to reach the send. Waiting on the log rather than on a fixed duration keeps + /// the test deterministic on a loaded machine, where scheduling a blocking task can + /// take far longer than a hard-coded sleep allows. + macro_rules! drive_until_scraping { + ($loop_fut:expr) => {{ + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !logs_contain("Scraping node metrics") { + assert!( + tokio::time::Instant::now() < deadline, + "the scrape loop never started a scrape" + ); + tokio::select! { + res = $loop_fut.as_mut() => panic!("scrape loop exited early: {:?}", res), + _ = tokio::time::sleep(Duration::from_millis(10)) => {} + } + } + // The scrape itself only reads the clock and calls uname, so this is ample. + tokio::select! { + res = $loop_fut.as_mut() => panic!("scrape loop exited early: {:?}", res), + _ = tokio::time::sleep(Duration::from_millis(100)) => {} + } + }}; + } + + /// A config that scrapes only cheap, always-available collectors + fn test_config() -> NodeMetricsConfig { + NodeMetricsConfig { + scrape_interval: Duration::from_secs(1), + collectors: vec![Collector::Uname, Collector::Time], + ..Default::default() + } + } + + /// [`test_config`] with a different scrape interval + /// + /// The interval also bounds the send (`SEND_TIMEOUT.min(scrape_interval)`), so it is + /// what makes the backpressure tests finish quickly. + fn test_config_with_interval(scrape_interval: Duration) -> NodeMetricsConfig { + NodeMetricsConfig { + scrape_interval, + ..test_config() + } + } + + /// A placeholder batch, used to occupy channel capacity + fn placeholder_message() -> payload::Message { + payload::Message::new(None, vec![], None) + } + + #[tokio::test] + async fn test_scrape_loop_cancellation() { + let (tx, _rx) = bounded::>(10); + let output = OTLPOutput::new(tx); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + + let handle = tokio::spawn(async move { + run_scrape_loop(test_config(), Some(output), cancel_clone).await + }); + + // Let it run briefly + tokio::time::sleep(Duration::from_millis(100)).await; + + cancel.cancel(); + + // Unwrap both layers: the join result and the loop's own Result, so a panicking + // or failing loop cannot pass this test. + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop task panicked") + .expect("scrape loop returned an error"); + } + + /// With no metrics pipeline, the loop must idle until cancelled rather than returning. + /// Returning early is what the agent interprets as an unexpected receiver exit, which + /// makes it shut the whole process down. + #[tokio::test] + async fn test_no_metrics_output_idles_until_cancelled() { + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + + let mut handle = + tokio::spawn(async move { run_scrape_loop(test_config(), None, cancel_clone).await }); + + // It must still be running after a moment + tokio::select! { + res = &mut handle => panic!("scrape loop exited before cancellation: {:?}", res), + _ = tokio::time::sleep(Duration::from_millis(200)) => {} + } + + cancel.cancel(); + + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop task panicked") + .expect("scrape loop returned an error"); + } + + /// A disconnected pipeline must surface as an error instead of being retried forever. + #[tokio::test] + async fn test_disconnected_pipeline_returns_error() { + let (tx, rx) = bounded::>(1); + let output = OTLPOutput::new(tx); + // Drop the receiving end so the first send fails + drop(rx); + + let cancel = CancellationToken::new(); + let result = tokio::time::timeout( + Duration::from_secs(5), + run_scrape_loop(test_config(), Some(output), cancel), + ) + .await + .expect("scrape loop did not return after the pipeline disconnected"); + + let err = result.expect_err("expected an error when the pipeline is disconnected"); + assert!( + err.to_string().contains("pipeline disconnected"), + "expected a disconnect error, got: {}", + err + ); + } + + /// The latch that stops a wedged scrape from leaking a blocking thread every + /// interval. Driving it directly avoids needing an unresponsive filesystem. + #[tokio::test] + async fn test_wedged_scrape_skips_cycles_until_it_finishes() { + // Nothing outstanding: scraping proceeds. + let mut outstanding: Option>> = None; + assert!(!scrape_still_running(&mut outstanding).await); + + // A task that never finishes stands in for a hung procfs read. + let (release, wedged) = tokio::sync::oneshot::channel::<()>(); + let mut outstanding = Some(tokio::spawn(async move { + let _ = wedged.await; + Vec::new() + })); + + // Every cycle while it runs must be skipped, and the handle retained. + for _ in 0..3 { + assert!( + scrape_still_running(&mut outstanding).await, + "a running scrape must cause the cycle to be skipped" + ); + assert!(outstanding.is_some(), "the handle must be retained"); + } + + // Once it completes, the latch clears and scraping resumes. + release.send(()).unwrap(); + let mut cleared = false; + for _ in 0..100 { + if !scrape_still_running(&mut outstanding).await { + cleared = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(cleared, "the latch never cleared after the scrape finished"); + assert!(outstanding.is_none(), "the finished handle must be reaped"); + } + + /// A panic in an abandoned scrape must be surfaced, not discarded with the handle. + #[traced_test] + #[tokio::test] + async fn test_wedged_scrape_panic_is_reaped() { + let handle = tokio::spawn(async { panic!("boom") }); + // Let it finish so the latch takes the reaping path + let _ = tokio::time::timeout(Duration::from_secs(5), async { + while !handle.is_finished() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + + let mut outstanding = Some(handle); + assert!(!scrape_still_running(&mut outstanding).await); + assert!(outstanding.is_none()); + + // Reaping is the only place the panic can be observed: the handle is dropped + // immediately afterwards, so a silent reap would lose it entirely. + assert!( + logs_contain("Previously timed-out scrape task failed"), + "the reaped panic must be logged" + ); + } + + /// A full pipeline must cost one batch, not the scrape loop. + /// + /// The send is bounded so that a backed-up exporter cannot park the loop forever: + /// the batch is dropped and the next scrape re-reads the current values. + #[traced_test] + #[tokio::test] + async fn test_full_pipeline_abandons_the_batch_and_keeps_scraping() { + // Capacity 1, pre-filled and never drained, so every send the loop attempts + // parks. Holding `_rx` keeps the channel connected: a dropped receiver would + // take the disconnect path instead. + let (tx, mut rx) = bounded::>(1); + tx.send(placeholder_message()).await.unwrap(); + let output = OTLPOutput::new(tx.clone()); + + // The send is bounded by min(SEND_TIMEOUT, scrape_interval), so the interval is + // also how long the first batch waits before being abandoned. + let interval = Duration::from_millis(300); + let cancel = CancellationToken::new(); + let mut loop_fut = std::pin::pin!(run_scrape_loop( + test_config_with_interval(interval), + Some(output), + cancel.clone() + )); + + // Drive until two batches have been abandoned, rather than timing a fixed window: + // the assertion below is about the count, so wait for the count. + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + loop { + let abandoned = logs_with!("abandoning this batch"); + if abandoned >= 2 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "only {} batches were abandoned before the deadline", + abandoned + ); + tokio::select! { + res = loop_fut.as_mut() => { + panic!("scrape loop exited while the pipeline was full: {:?}", res) + } + _ = tokio::time::sleep(Duration::from_millis(20)) => {} + } + } + + // Warned, not silently dropped: this is the only signal that the exporter is + // backed up badly enough to be losing data. + assert!( + logs_contain("abandoning this batch"), + "an abandoned batch must be reported" + ); + + // More than one batch was abandoned, so the loop went on scraping instead of + // parking on the full channel for good after the first. + logs_assert(|lines: &[&str]| { + let abandoned = lines + .iter() + .filter(|line| line.contains("abandoning this batch")) + .count(); + if abandoned >= 2 { + Ok(()) + } else { + Err(format!( + "expected at least 2 abandoned batches, saw {}", + abandoned + )) + } + }); + + // The batch really was dropped rather than queued behind the placeholder. + // Draining shows the abandoned batches were never queued: only the placeholder is + // there. (Asserting `tx.len() == 1` would be vacuous — capacity is 1.) + let queued = rx.next().await.expect("channel closed"); + assert!( + queued.payload.is_empty(), + "only the placeholder should have been queued, got {} payload(s)", + queued.payload.len() + ); + + // The loop is still alive and still returns cleanly. + cancel.cancel(); + tokio::time::timeout(Duration::from_secs(5), loop_fut) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop returned an error"); + } + + /// Cancellation gives an in-flight send a bounded window to complete. + /// + /// The pipeline is still draining when receivers are cancelled, so a batch that can + /// still be handed off should be, rather than discarded outright. + #[traced_test] + #[tokio::test] + async fn test_shutdown_grace_window_lets_an_inflight_send_finish() { + // Pre-fill the single slot so the loop's first send parks, and use an interval + // long enough that it is certainly still parked when cancellation arrives. + let (tx, mut rx) = bounded::>(1); + tx.send(placeholder_message()).await.unwrap(); + let output = OTLPOutput::new(tx.clone()); + + let cancel = CancellationToken::new(); + let mut loop_fut = std::pin::pin!(run_scrape_loop( + test_config_with_interval(Duration::from_secs(30)), + Some(output), + cancel.clone() + )); + + // Let the first scrape finish and park in the send. + drive_until_scraping!(loop_fut); + + cancel.cancel(); + + // Freeing the slot lets the parked send complete inside SHUTDOWN_SEND_TIMEOUT. + let placeholder = rx.next().await.expect("channel closed"); + assert!( + placeholder.payload.is_empty(), + "expected the pre-filled placeholder to be drained first" + ); + + // The loop must return well inside the agent's 3 second receiver drain budget. + let started = tokio::time::Instant::now(); + tokio::time::timeout(Duration::from_secs(3), loop_fut) + .await + .expect("scrape loop did not stop within the receiver drain budget") + .expect("scrape loop returned an error"); + let elapsed = started.elapsed(); + + // The batch was handed off during the grace window, not dropped. + assert!( + logs_contain("metric points during shutdown"), + "the final batch should have been sent during shutdown" + ); + let batch = tokio::time::timeout(Duration::from_secs(1), rx.next()) + .await + .expect("timed out waiting for the final batch") + .expect("channel closed without the final batch"); + assert_eq!(batch.payload.len(), 1); + assert!( + !batch.payload[0].scope_metrics[0].metrics.is_empty(), + "the final batch must carry the scraped metrics" + ); + + // Completing early is the point: the window is a bound, not a delay. + assert!( + elapsed < SHUTDOWN_SEND_TIMEOUT, + "shutdown took {:?}, which means the send did not complete in the window", + elapsed + ); + } + + /// A scrape whose metrics are all filtered out must not send an empty batch. + #[traced_test] + #[tokio::test] + async fn test_fully_filtered_scrape_sends_nothing() { + let config = NodeMetricsConfig { + scrape_interval: Duration::from_secs(1), + collectors: vec![Collector::Uname, Collector::Time], + ..Default::default() + } + .with_exclude_filter(".*") + .expect("valid regex"); + + let (tx, mut rx) = bounded::>(10); + let output = OTLPOutput::new(tx); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + let handle = + tokio::spawn(async move { run_scrape_loop(config, Some(output), cancel_clone).await }); + + // Nothing should arrive on the channel + let got = tokio::time::timeout(Duration::from_millis(500), rx.next()).await; + assert!( + got.is_err(), + "expected no message for a fully filtered scrape" + ); + + // ...and the absence of a message must be because the loop filtered everything + // out, not because it died on its first iteration. + let mut handle = handle; + tokio::select! { + res = &mut handle => panic!("scrape loop exited unexpectedly: {:?}", res), + _ = tokio::time::sleep(Duration::from_millis(50)) => {} + } + + // An empty scrape means a wrong procfs/sysfs path or an over-broad filter, and + // is otherwise invisible: nothing arrives on the channel either way. + assert!( + logs_contain("No metrics collected"), + "an empty scrape must be reported" + ); + + cancel.cancel(); + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop task panicked") + .expect("scrape loop returned an error"); + } + + /// Integration test: run a full scrape→convert→send cycle and verify + /// that ResourceMetrics actually arrive on the output channel. + /// Uses only the Uname collector which works on all unix platforms + /// (libc::uname + SystemTime::now). + #[tokio::test] + async fn test_scrape_produces_metrics_on_channel() { + let config = NodeMetricsConfig { + scrape_interval: Duration::from_secs(1), + collectors: vec![Collector::Uname, Collector::Time], + ..Default::default() + }; + + let (tx, mut rx) = bounded::>(10); + let output = OTLPOutput::new(tx); + + let cancel = CancellationToken::new(); + let cancel_clone = cancel.clone(); + + let handle = + tokio::spawn(async move { run_scrape_loop(config, Some(output), cancel_clone).await }); + + // Wait for the first scrape to produce a message + let msg = tokio::time::timeout(Duration::from_secs(5), rx.next()) + .await + .expect("timed out waiting for metrics") + .expect("channel closed without receiving metrics"); + + cancel.cancel(); + + // Unwrap both layers, like the sibling tests: a loop that produced one good + // batch and then failed must not pass. + tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("scrape loop did not stop after cancellation") + .expect("scrape loop task panicked") + .expect("scrape loop returned an error"); + + // Verify the message contains ResourceMetrics with actual data + assert_eq!(msg.payload.len(), 1); + let resource_metrics = &msg.payload[0]; + + // Should have a resource with service.name + let resource = resource_metrics + .resource + .as_ref() + .expect("missing resource"); + assert!( + resource + .attributes + .iter() + .any(|kv| kv.key == "service.name") + ); + + // Should have node_uname_info (from Uname) and node_time_seconds (from Time) + let scope_metrics = &resource_metrics.scope_metrics; + assert_eq!(scope_metrics.len(), 1); + let metric_names: Vec<&str> = scope_metrics[0] + .metrics + .iter() + .map(|m| m.name.as_str()) + .collect(); + for expected in ["node_time_seconds", "node_uname_info"] { + assert!( + metric_names.contains(&expected), + "expected {} in {:?}", + expected, + metric_names + ); + } + + // host.name must be on the resource so multi-host series stay distinguishable + assert!(resource.attributes.iter().any(|kv| kv.key == "host.name")); + } +} diff --git a/tests/node_metrics_integration_tests.rs b/tests/node_metrics_integration_tests.rs new file mode 100644 index 00000000..12f8b90b --- /dev/null +++ b/tests/node_metrics_integration_tests.rs @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! Node Metrics Integration Tests +//! +//! These tests run against the real `/proc` and `/sys` of a Linux host. They exist to +//! catch what the fixture-based unit tests cannot: a drift between the kernel's actual +//! file formats and the parsers in this receiver. +//! +//! To run these tests: +//! +//! ``` +//! NODE_METRICS_INTEGRATION_TESTS=true cargo test --test node_metrics_integration_tests --features node_metrics_receiver +//! ``` +//! +//! Note: collectors for optional hardware (thermal zones, hwmon sensors, NVMe devices, +//! cpufreq, diskstats) are absent in many containers and virtual machines. For those the +//! tests accept an empty result and require only that any value produced is finite; they +//! never require a particular piece of hardware to be present, which would be flaky by +//! design. + +#![cfg(all( + target_os = "linux", + node_metrics_integration_tests = "true", + feature = "node_metrics_receiver" +))] + +use rotel::receivers::node_metrics::collector::{MetricType, SystemCollector}; +use rotel::receivers::node_metrics::config::{Collector, NodeMetricsConfig}; + +/// Build a config restricted to the given collectors, reading the real procfs/sysfs +fn config_for(collectors: Vec) -> NodeMetricsConfig { + NodeMetricsConfig { + collectors, + ..Default::default() + } +} + +#[test] +fn test_boot_time_is_plausible() { + let collector = SystemCollector::default(); + let boot_time = collector.boot_time(); + + // Deliberately tolerant: a board with no RTC reports a near-epoch btime until NTP + // syncs, and the collector itself treats 0 as "unknown" and suppresses the metric. So + // only require that the value is not in the future. + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + assert!( + boot_time <= now, + "boot time {} is in the future (now {})", + boot_time, + now + ); +} + +#[test] +fn test_core_collectors_produce_metrics_on_a_real_host() { + let collector = SystemCollector::default(); + let config = config_for(vec![ + Collector::Cpu, + Collector::Loadavg, + Collector::Memory, + Collector::Filesystem, + Collector::Stat, + Collector::Vmstat, + ]); + + let metrics = collector.collect(&config); + assert!( + !metrics.is_empty(), + "expected metrics from a real Linux host" + ); + + // These four exist on every Linux kernel this receiver targets. If the kernel ever + // changes a format the parser depends on, one of them disappears. + // node_filesystem_size_bytes is deliberately absent from this list: a container whose + // only mounts are overlay/proc/sysfs reports no filesystems at all, and those types are + // excluded by design. test_filesystem_stats_are_self_consistent covers it where present. + for expected in [ + "node_cpu_seconds_total", + "node_memory_MemTotal_bytes", + "node_load1", + ] { + assert!( + metrics.iter().any(|m| m.name == expected), + "missing {} from a real host scrape", + expected + ); + } + + // The payoff of running against a real kernel: no value is ever NaN or infinite, + // whatever the host looks like. + for m in &metrics { + assert!( + m.value.is_finite(), + "metric {} has a non-finite value: {}", + m.name, + m.value + ); + // Non-negativity is only asserted where it is part of the metric's definition — + // byte, second and filesystem gauges. Kernel-supplied fields in general (and + // temperatures in particular) may legitimately be negative. + let must_be_non_negative = m.name.ends_with("_bytes") + || m.name.ends_with("_bytes_total") + || m.name.ends_with("_seconds_total") + || m.name.starts_with("node_filesystem_") + || m.name.starts_with("node_load"); + if must_be_non_negative { + assert!( + m.value >= 0.0, + "metric {} has a negative value: {}", + m.name, + m.value + ); + } + } +} + +#[test] +fn test_filesystem_stats_are_self_consistent() { + let collector = SystemCollector::default(); + let metrics = collector.collect(&config_for(vec![Collector::Filesystem])); + + // Do not require a mount point of "/": inside a container the root is usually an + // overlay mount, which is excluded by filesystem type. Pick the largest reported + // filesystem instead, which exists on any host that reports one at all. + let Some(labels) = metrics + .iter() + .filter(|m| m.name == "node_filesystem_size_bytes") + .max_by(|a, b| a.value.total_cmp(&b.value)) + .map(|m| m.labels.clone()) + else { + eprintln!("Skipping test: no filesystems reported on this host"); + return; + }; + let mountpoint = labels + .iter() + .find(|(k, _)| k == "mountpoint") + .map(|(_, v)| v.as_str()) + .unwrap_or(""); + + // Key on the whole label set, not just the mount point: with an over-mount two + // devices share a mount point, and mixing their values would fail spuriously. + let value_for = |name: &str| -> Option { + metrics + .iter() + .find(|m| m.name == name && m.labels == labels) + .map(|m| m.value) + }; + + let size = value_for("node_filesystem_size_bytes").expect("no size"); + let free = value_for("node_filesystem_free_bytes").expect("no free"); + let avail = value_for("node_filesystem_avail_bytes").expect("no avail"); + + assert!( + size > 0.0, + "filesystem {} size should be positive", + mountpoint + ); + assert!( + avail <= free && free <= size, + "{}: expected avail ({}) <= free ({}) <= size ({})", + mountpoint, + avail, + free, + size + ); +} + +#[test] +fn test_counters_and_gauges_are_classified_on_a_real_host() { + let collector = SystemCollector::default(); + let metrics = collector.collect(&config_for(vec![Collector::Cpu, Collector::Stat])); + + let cpu = metrics + .iter() + .find(|m| m.name == "node_cpu_seconds_total") + .expect("no CPU metric"); + assert_eq!(cpu.metric_type, MetricType::Counter); + + let running = metrics + .iter() + .find(|m| m.name == "node_procs_running") + .expect("no node_procs_running metric"); + assert_eq!(running.metric_type, MetricType::Gauge); +} + +/// Collectors backed by interfaces every Linux kernel provides must return something. +/// If one silently stops parsing, this is what notices. +#[test] +fn test_always_available_collectors_produce_metrics() { + let collector = SystemCollector::default(); + + // Netstat, Sockstat and Filefd are deliberately excluded: they need + // /proc/net/{netstat,snmp,sockstat} and /proc/sys/fs/file-nr, which sandboxes such as + // gVisor and WSL1 only partially synthesise. They are covered by the tolerate-empty + // test below instead. + for c in [ + Collector::Network, + Collector::Uname, + Collector::Time, + Collector::Processes, + ] { + let metrics = collector.collect(&config_for(vec![c.clone()])); + assert!( + !metrics.is_empty(), + "collector {:?} produced no metrics on a real Linux host", + c + ); + for m in &metrics { + assert!( + m.value.is_finite(), + "collector {:?} produced a non-finite value for {}", + c, + m.name + ); + } + } +} + +/// Hardware-dependent collectors may legitimately produce nothing (container, VM, no +/// sensors), so assert only that they degrade quietly. Note these can report negative +/// values — sub-zero temperatures are legal — so no sign assertion here. +#[test] +fn test_optional_hardware_collectors_do_not_panic() { + let collector = SystemCollector::default(); + + for c in [ + Collector::Cpufreq, + Collector::ThermalZone, + Collector::Nvme, + Collector::Hwmon, + Collector::Diskstats, + // Present on ordinary kernels but only partially synthesised by some sandboxes + Collector::Netstat, + Collector::Sockstat, + Collector::Filefd, + ] { + let metrics = collector.collect(&config_for(vec![c.clone()])); + for m in &metrics { + assert!( + m.value.is_finite(), + "collector {:?} produced a non-finite value for {}", + c, + m.name + ); + } + } +} + +/// The invariant a fixture can never prove: on a real host's mount table, hwmon set and +/// network devices, no OTLP metric may contain two data points with the same attribute +/// set, and no (name, type) pair may appear twice. +#[test] +fn test_real_host_scrape_has_no_conflicting_data_points() { + use rotel::receivers::node_metrics::convert::convert_to_otlp_metrics; + + let collector = SystemCollector::default(); + let metrics = collector.collect(&NodeMetricsConfig::default()); + let resource_metrics = convert_to_otlp_metrics(metrics, collector.boot_time(), "node_metrics"); + + let mut seen_metrics = std::collections::HashSet::new(); + for sm in &resource_metrics.scope_metrics { + for m in &sm.metrics { + let points = match &m.data { + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(g)) => { + assert!( + seen_metrics.insert((m.name.clone(), "gauge")), + "duplicate gauge {}", + m.name + ); + &g.data_points + } + Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(s)) => { + assert!( + seen_metrics.insert((m.name.clone(), "sum")), + "duplicate sum {}", + m.name + ); + &s.data_points + } + _ => continue, + }; + + let mut seen_points = std::collections::HashSet::new(); + for dp in points { + let mut key: Vec<(String, String)> = dp + .attributes + .iter() + .map(|kv| { + let value = match &kv.value { + Some(v) => format!("{:?}", v.value), + None => String::new(), + }; + (kv.key.clone(), value) + }) + .collect(); + key.sort(); + assert!( + seen_points.insert(key.clone()), + "metric {} has duplicate data points for attributes {:?}", + m.name, + key + ); + } + } + } +} + +#[test] +fn test_hwmon_chip_labels_are_unique_per_sensor() { + // Two chips of the same model (e.g. two NVMe drives) must not collapse onto the same + // chip label, which would produce duplicate OTLP attribute sets. + let collector = SystemCollector::default(); + let metrics = collector.collect(&config_for(vec![Collector::Hwmon])); + + let mut seen = std::collections::HashSet::new(); + for m in metrics.iter().filter(|m| m.name != "node_hwmon_chip_names") { + let key = (m.name.clone(), m.labels.clone()); + assert!( + seen.insert(key), + "duplicate label set for {}: {:?}", + m.name, + m.labels + ); + } +}