From f099afccdeffd71aac6cf558a42d2818f338d342 Mon Sep 17 00:00:00 2001 From: "Christopher M. Cantalupo" Date: Wed, 23 Sep 2026 13:31:17 -0700 Subject: [PATCH 1/6] pkg/monitor: report AET activity in nanofarads The kernel reports the AET "activity" counter in nanofarads, not farads as the resctrl documentation currently states (a documentation fix is in progress). Reading.Unit for activity was "farads", mislabeling the raw value by a factor of 1e9. Record the raw unit as UCUM "nF" so ReadCounters stays faithful to the kernel value. For OTel export, convert to the base unit: the perf.activity instrument keeps unit "farads" (otlptranslator has no suffix mapping for UCUM "F", so "farads" keeps the Prometheus name perf_activity_farads_total) and observed values are scaled by 1e-9. Scaling is applied after the monotonic accumulator so its state stays in kernel units. Signed-off-by: Christopher M. Cantalupo --- doc/resctrl-mon.md | 2 +- pkg/monitor/otel.go | 10 +++++++--- pkg/monitor/reading.go | 27 +++++++++++++++++++++++---- pkg/monitor/reading_test.go | 20 ++++++++++++++++++++ 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/doc/resctrl-mon.md b/doc/resctrl-mon.md index 9014cfd..bd733c2 100644 --- a/doc/resctrl-mon.md +++ b/doc/resctrl-mon.md @@ -114,7 +114,7 @@ mgr.Reconcile(liveKeys) readings, _ := mgr.ReadCounters(podUID) for _, r := range readings { // r.Kind: monitor.Gauge (instantaneous) or monitor.Cumulative (monotonic counter) - // r.Unit: UCUM where available ("By", "J"), otherwise descriptive ("farads", "") + // r.Unit: UCUM unit of the raw kernel value where known ("By", "J", "nF"), otherwise "" fmt.Printf("%s/%s = %f (%v, %s)\n", r.Domain, r.Name, r.Value, r.Kind, r.Unit) } ``` diff --git a/pkg/monitor/otel.go b/pkg/monitor/otel.go index 8479d14..a7ca766 100644 --- a/pkg/monitor/otel.go +++ b/pkg/monitor/otel.go @@ -107,8 +107,8 @@ func (r *Registration) Close() error { // mon_L3_00/llc_occupancy → l3.llc.occupancy (unit: By) // mon_L3_00/mbm_local_bytes → l3.mbm.local.bytes (unit: By) // mon_L3_00/mbm_total_bytes → l3.mbm.total.bytes (unit: By) -// mon_PERF_PKG_00/core_energy → perf.core.energy (unit: J) -// mon_PERF_PKG_00/activity → perf.activity (unit: farads) +// mon_PERF_PKG_00/core_energy → perf.core.energy (unit: J) +// mon_PERF_PKG_00/activity → perf.activity (unit: farads; kernel nF × 1e-9) // mon_PERF_PKG_00/c1_res → perf.c1.res // mon_PERF_PKG_00/c6_res → perf.c6.res // mon_PERF_PKG_00/uops_retired → perf.uops.retired @@ -202,7 +202,8 @@ func (o *otelObserver) discoverAndRegister() error { continue } seen[name] = struct{}{} - instr, err := o.createInstrument(name, metaKind(c.counter), metaUnit(c.counter)) + unit, _ := metaOTel(c.counter) + instr, err := o.createInstrument(name, metaKind(c.counter), unit) if err != nil { return err } @@ -316,6 +317,9 @@ func (o *otelObserver) observe(ctx context.Context, obs metric.Observer) { if r.Kind == Cumulative { val = o.accum.monotonic(key, r.Domain, r.Name, val, g.Gen()) } + // Scale after accumulating so accumulator state stays in kernel units. + _, scale := metaOTel(r.Name) + val *= scale attrs := make([]attribute.KeyValue, 0, len(groupAttrs)+2) attrs = append(attrs, attribute.String("domain.id", DomainInstance(r.Domain))) diff --git a/pkg/monitor/reading.go b/pkg/monitor/reading.go index cca0158..36b596c 100644 --- a/pkg/monitor/reading.go +++ b/pkg/monitor/reading.go @@ -46,7 +46,7 @@ type Reading struct { Name string // counter file name, e.g. "llc_occupancy", "core_energy" Value float64 // parsed value (float to cover core_energy/activity) Kind ReadingKind // Gauge or Cumulative - Unit string // UCUM unit where available (e.g. "By", "J"), otherwise descriptive (e.g. "farads") + Unit string // UCUM unit of Value where known (e.g. "By", "J", "nF"), otherwise "" } // readingMeta maps known counter names to their kind and unit. @@ -58,9 +58,9 @@ var readingMeta = map[string]struct { "mbm_total_bytes": {Cumulative, "By"}, "mbm_local_bytes": {Cumulative, "By"}, "core_energy": {Cumulative, "J"}, - // activity accumulates dynamic capacitance; its rate of change is the - // workload's dynamic capacitance (Cdyn) - "activity": {Cumulative, "farads"}, + // activity accumulates dynamic capacitance in nanofarads; its rate of + // change is the workload's dynamic capacitance (Cdyn) + "activity": {Cumulative, "nF"}, "c1_res": {Cumulative, ""}, "c6_res": {Cumulative, ""}, "uops_retired": {Cumulative, ""}, @@ -163,3 +163,22 @@ func metaUnit(name string) string { } return "" } + +// otelExport overrides the exported OTel unit and scale for counters whose raw +// kernel unit is not a base unit. +var otelExport = map[string]struct { + unit string + scale float64 +}{ + // "farads" rather than UCUM "F": otlptranslator has no farad suffix mapping. + "activity": {"farads", 1e-9}, +} + +// metaOTel returns the OTel instrument unit for a counter and the factor that +// converts its raw value into that unit. +func metaOTel(name string) (unit string, scale float64) { + if o, ok := otelExport[name]; ok { + return o.unit, o.scale + } + return metaUnit(name), 1 +} diff --git a/pkg/monitor/reading_test.go b/pkg/monitor/reading_test.go index a0f2953..9a7a3e3 100644 --- a/pkg/monitor/reading_test.go +++ b/pkg/monitor/reading_test.go @@ -50,8 +50,10 @@ func TestReadCounters_MultiDomain(t *testing.T) { // Build a lookup map: domain/name -> value type key struct{ domain, name string } got := make(map[key]float64) + units := make(map[key]string) for _, r := range readings { got[key{r.Domain, r.Name}] = r.Value + units[key{r.Domain, r.Name}] = r.Unit } // L3 domain — integer values parse as float64. @@ -62,6 +64,11 @@ func TestReadCounters_MultiDomain(t *testing.T) { assert.InDelta(t, 54446119.644974, got[key{"mon_PERF_PKG_00", "core_energy"}], 0.001) assert.InDelta(t, 1042.371582, got[key{"mon_PERF_PKG_00", "activity"}], 0.001) + // Readings carry the raw kernel value and unit (activity is nanofarads). + assert.Equal(t, "J", units[key{"mon_PERF_PKG_00", "core_energy"}]) + assert.Equal(t, "nF", units[key{"mon_PERF_PKG_00", "activity"}]) + assert.Equal(t, "By", units[key{"mon_L3_00", "mbm_total_bytes"}]) + // mon_L3_01 has only "Unavailable" — no reading should be emitted for it. _, hasL301 := got[key{"mon_L3_01", "llc_occupancy"}] assert.False(t, hasL301, "non-numeric 'Unavailable' should be skipped") @@ -197,3 +204,16 @@ func TestMetaKind_UnknownDefaultsToGauge(t *testing.T) { // accumulator rather than risk corrupting a real gauge. assert.Equal(t, Gauge, metaKind("some_future_counter")) } + +func TestMetaOTel(t *testing.T) { + // Counters listed in otelExport are exported in its unit and scale. + for name, want := range otelExport { + unit, scale := metaOTel(name) + assert.Equal(t, want.unit, unit, name) + assert.Equal(t, want.scale, scale, name) + } + // Other counters are exported in their raw unit, unscaled. + unit, scale := metaOTel("core_energy") + assert.Equal(t, metaUnit("core_energy"), unit) + assert.Equal(t, 1.0, scale) +} From deb8a663b5fc24089356e8ab1a94af43002070ce Mon Sep 17 00:00:00 2001 From: "Christopher M. Cantalupo" Date: Thu, 24 Sep 2026 15:34:43 -0700 Subject: [PATCH 2/6] pkg/monitor: correct Prometheus naming documentation The InstrumentName documentation claimed that keeping the _bytes suffix makes the OTel Prometheus bridge render mbm_total_bytes as l3_mbm_total_bytes_total, and that pkg/rdt's l3.mbm.total renders incorrectly as l3_mbm_bytes_total. The rendered name is actually chosen by the consumer's exporter translation strategy: UnderscoreEscapingWithSuffixes -> l3_mbm_bytes_total NoUTF8EscapingWithSuffixes -> l3.mbm.total.bytes_total The underscore strategy removes every "total" word from a counter name before appending _total, so pkg/rdt's name renders identically. Older exporter releases derived their default strategy from model.NameValidationScheme (UTF-8, hence the NoUTF8 form, which a legacy scrape escapes to l3_mbm_total_bytes_total); newer releases default to the underscore strategy. Rewrite the documentation to describe the mechanical file-to-instrument mapping and the strategy-dependent rendering, and recommend that consumers select a strategy explicitly. Signed-off-by: Christopher M. Cantalupo --- pkg/monitor/otel.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/pkg/monitor/otel.go b/pkg/monitor/otel.go index a7ca766..2f0d36c 100644 --- a/pkg/monitor/otel.go +++ b/pkg/monitor/otel.go @@ -131,7 +131,8 @@ func (r *Registration) Close() error { // L3 instrument names are similar to pkg/rdt's RegisterOpenTelemetryInstruments // but preserve the _bytes counter suffix (e.g. l3.mbm.total.bytes vs // pkg/rdt's l3.mbm.total). This maintains backward compatibility with the -// kernel counter file names as a mechanical derivation. +// kernel counter file names as a mechanical derivation. See InstrumentName for +// how Prometheus exporters render these names. // // Each metric carries a "domain.id" attribute with the numeric instance // (e.g. "00") and a "domain.name" attribute with the full domain directory @@ -347,21 +348,22 @@ func (o *otelObserver) observe(ctx context.Context, obs metric.Observer) { // --- Naming helpers (exported for use by callers building custom export) --- // InstrumentName derives the OTel instrument name from a resctrl domain -// directory name and counter file name. +// directory name and counter file name: the domain's resource prefix followed +// by the counter file name with "_" replaced by ".". The mapping is mechanical +// (the _bytes suffix is kept) so every instrument name identifies its resctrl +// file. // -// The counter file name is converted to dot-separated segments and prepended -// with the domain's resource prefix. The _bytes suffix is preserved (not -// stripped) so that the OTel→Prometheus bridge's unit-suffix deduplication -// produces correct names without colliding with the counter _total suffix -// convention. +// Prometheus names are chosen by the consumer's exporter, not by this package, +// and depend on its translation strategy. For the counter l3.mbm.total.bytes +// (unit By): // -// NOTE: This intentionally diverges from pkg/rdt's RegisterOpenTelemetryInstruments -// which uses names like "l3.mbm.total" (stripping _bytes). That approach -// produces incorrect Prometheus names via the OTel bridge: the bridge treats -// the trailing "total" as a counter suffix, yielding "l3_mbm_bytes_total" -// instead of the expected "l3_mbm_total_bytes_total". By preserving _bytes in -// the OTel name, the bridge sees the unit is already present and only appends -// _total for counters, producing the correct final name. +// UnderscoreEscapingWithSuffixes → l3_mbm_bytes_total +// NoUTF8EscapingWithSuffixes → l3.mbm.total.bytes_total +// +// The underscore strategy removes every "total" word from a counter name +// before appending _total, so pkg/rdt's l3.mbm.total renders the same way. +// Consumers should select a strategy explicitly, e.g. with +// prometheus.WithTranslationStrategy(otlptranslator.UnderscoreEscapingWithSuffixes). // // Examples: // From 9bc9e33005bbcf22e93be525c0c96b020ecba9e8 Mon Sep 17 00:00:00 2001 From: "Christopher M. Cantalupo" Date: Wed, 23 Sep 2026 13:33:42 -0700 Subject: [PATCH 3/6] pkg/monitor: warn once per undiscovered counter observe logged a warning for every reading of a counter that was not discovered at registration, i.e. once per group per collection. With frequent Prometheus scrapes and many groups this floods the log. Warn the first time an undiscovered instrument name is seen and log subsequent occurrences at debug level. Signed-off-by: Christopher M. Cantalupo --- pkg/monitor/otel.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pkg/monitor/otel.go b/pkg/monitor/otel.go index 2f0d36c..df5498e 100644 --- a/pkg/monitor/otel.go +++ b/pkg/monitor/otel.go @@ -158,6 +158,7 @@ func (m *Manager) RegisterOTelInstruments(meter metric.Meter, opts ...OTelOption cfg: cfg, meter: meter, instrs: make(map[string]metric.Observable), + warned: make(map[string]bool), accum: newOTelAccumulator(), } @@ -178,6 +179,7 @@ type otelObserver struct { mu sync.Mutex instrs map[string]metric.Observable // instrName → instrument + warned map[string]bool // undiscovered instrument names already warned about accum *otelAccumulator reg metric.Registration // batch callback registration; nil if none } @@ -307,9 +309,17 @@ func (o *otelObserver) observe(ctx context.Context, obs metric.Observer) { instrName := InstrumentName(r.Domain, r.Name) o.mu.Lock() instr := o.instrs[instrName] + warned := o.warned[instrName] + if instr == nil && !warned { + o.warned[instrName] = true + } o.mu.Unlock() if instr == nil { - log().Warn("otel: unknown counter skipped (not discovered at registration)", + logUnknown := log().Debug + if !warned { + logUnknown = log().Warn + } + logUnknown("otel: unknown counter skipped (not discovered at registration)", "instrument", instrName, "domain", r.Domain, "counter", r.Name) continue } From 5b5d9b0f5bf5f99747634685cda8d00f283ad4f4 Mon Sep 17 00:00:00 2001 From: "Christopher M. Cantalupo" Date: Wed, 23 Sep 2026 13:34:03 -0700 Subject: [PATCH 4/6] pkg/monitor: warn when no counters are discovered RegisterOTelInstruments silently returned a no-op Registration when the resctrl root had no readable mon_data counters (for example when resctrl is mounted without monitoring support), leaving the consumer exporting nothing with no indication why. Log a warning naming the scanned mon_data path. The call still succeeds and returns a no-op Registration, so the behavior stays compatible. Signed-off-by: Christopher M. Cantalupo --- pkg/monitor/otel.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/monitor/otel.go b/pkg/monitor/otel.go index df5498e..8378d0c 100644 --- a/pkg/monitor/otel.go +++ b/pkg/monitor/otel.go @@ -190,7 +190,8 @@ type otelObserver struct { // not read — so instruments are registered even when the kernel reports // temporary placeholder values like "Unavailable". func (o *otelObserver) discoverAndRegister() error { - counters, err := discoverCounters(filepath.Join(o.mgr.root, "mon_data")) + monDataPath := filepath.Join(o.mgr.root, "mon_data") + counters, err := discoverCounters(monDataPath) if err != nil { return err } @@ -215,6 +216,7 @@ func (o *otelObserver) discoverAndRegister() error { } if len(observables) == 0 { + log().Warn("otel: no resctrl counters discovered; nothing will be exported", "path", monDataPath) return nil } From d4e20831efe872a5d3d1a475b391352f2e5b8a89 Mon Sep 17 00:00:00 2001 From: "Christopher M. Cantalupo" Date: Wed, 23 Sep 2026 13:34:23 -0700 Subject: [PATCH 5/6] pkg/monitor: log routine group lifecycle at debug level Creating a group, assigning a PID and removing a group are routine per-workload operations that consumers already log in their own terms (for example per pod). Logging them at info level in the library duplicates that output and ignores the consumer's chosen verbosity. Log these three events at debug level. Adopting a pre-existing group and reaping an orphan in Reconcile stay at info level, and all warnings are unchanged. Signed-off-by: Christopher M. Cantalupo --- pkg/monitor/monitor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/monitor/monitor.go b/pkg/monitor/monitor.go index 2b678b1..3a7f336 100644 --- a/pkg/monitor/monitor.go +++ b/pkg/monitor/monitor.go @@ -294,7 +294,7 @@ func (m *Manager) EnsureGroup(key, rdtClass string) (*Group, error) { return nil, fmt.Errorf("failed to create mon_group %s: %w", monGroupDir, err) } } else { - log().Info("created mon_group", "key", key, "dir", monGroupDir) + log().Debug("created mon_group", "key", key, "dir", monGroupDir) } // Bump the monotonic generation on every (re)creation so a remove→recreate @@ -363,7 +363,7 @@ func (m *Manager) AssignPID(key string, pid int) error { if err := f.Close(); err != nil { return fmt.Errorf("failed to write pid %d for key %s: %w", pid, key, err) } - log().Info("assigned PID to mon_group", "key", key, "pid", pid) + log().Debug("assigned PID to mon_group", "key", key, "pid", pid) return nil } @@ -485,7 +485,7 @@ func (m *Manager) Remove(key string) error { } delete(m.entries, key) - log().Info("removed mon_group", "key", key, "dir", e.dir) + log().Debug("removed mon_group", "key", key, "dir", e.dir) return nil } From 57ea66f71469781d5e7ba3ce4cf314cfabb4bf0d Mon Sep 17 00:00:00 2001 From: "Christopher M. Cantalupo" Date: Wed, 23 Sep 2026 13:34:50 -0700 Subject: [PATCH 6/6] doc: resctrl-mon guidance for teardown, units and Prometheus names - Teardown: remove a pod's group only when its last sandbox is gone. Kubelet can create a new sandbox for the same pod UID and garbage-collect the old one while the pod keeps running, so removing the group on every sandbox removal would delete a live group. - Add an OpenTelemetry Export section listing each counter's instrument name, kind, exported unit and value (activity is exported in farads, scaled from the kernel's nanofarads). - Document the Prometheus names under both exporter translation strategies and recommend selecting a strategy explicitly. Signed-off-by: Christopher M. Cantalupo --- doc/resctrl-mon.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/doc/resctrl-mon.md b/doc/resctrl-mon.md index bd733c2..6e02860 100644 --- a/doc/resctrl-mon.md +++ b/doc/resctrl-mon.md @@ -105,6 +105,9 @@ mgr.AssignPID(podUID, pid) // Tie the group's lifetime to the pod, not to individual containers — a // container restart reuses the sandbox, and releasing the RMID early would // hand the replacement a fresh RMID whose counters carry residual values. +// Remove only when the pod's last sandbox is gone: kubelet can create a new +// sandbox for the same pod UID and garbage-collect the old one while the pod +// keeps running. mgr.Remove(podUID) // Crash recovery: remove orphaned groups not in the live set @@ -119,6 +122,48 @@ for _, r := range readings { } ``` +## OpenTelemetry Export + +`Manager.RegisterOTelInstruments` registers one observable instrument per +counter discovered under the resctrl root's `mon_data`. Instrument names are +derived mechanically from the counter files (see `InstrumentName`), cumulative +counters pass through a monotonic accumulator, and every data point carries +`domain.id` and `domain.name` attributes. + +| resctrl file | Instrument | Kind | Unit | Exported value | +|---|---|---|---|---| +| `mon_L3_*/llc_occupancy` | `l3.llc.occupancy` | gauge | `By` | raw | +| `mon_L3_*/mbm_local_bytes` | `l3.mbm.local.bytes` | counter | `By` | raw | +| `mon_L3_*/mbm_total_bytes` | `l3.mbm.total.bytes` | counter | `By` | raw | +| `mon_PERF_PKG_*/core_energy` | `perf.core.energy` | counter | `J` | raw | +| `mon_PERF_PKG_*/activity` | `perf.activity` | counter | `farads` | raw nanofarads × 1e-9 | +| other `mon_PERF_PKG_*` files | `perf.` | counter if known, else gauge | none | raw | + +`ReadCounters` always returns the raw kernel value; for `activity` its `Unit` +is `nF`. + +Prometheus names are produced by the consumer's exporter and depend on its +translation strategy: + +| Instrument | `UnderscoreEscapingWithSuffixes` | `NoUTF8EscapingWithSuffixes` | +|---|---|---| +| `l3.llc.occupancy` | `l3_llc_occupancy_bytes` | `l3.llc.occupancy_bytes` | +| `l3.mbm.local.bytes` | `l3_mbm_local_bytes_total` | `l3.mbm.local.bytes_total` | +| `l3.mbm.total.bytes` | `l3_mbm_bytes_total` | `l3.mbm.total.bytes_total` | +| `perf.core.energy` | `perf_core_energy_joules_total` | `perf.core.energy_joules_total` | +| `perf.activity` | `perf_activity_farads_total` | `perf.activity_farads_total` | + +The underscore strategy drops the word `total` from `l3.mbm.total.bytes`. +Older `go.opentelemetry.io/otel/exporters/prometheus` releases choose their +default strategy from the global name validation scheme, so select one +explicitly: + +```go +exp, err := prometheus.New( + prometheus.WithTranslationStrategy(otlptranslator.UnderscoreEscapingWithSuffixes), +) +``` + ## RMID Exhaustion Each mon_group consumes one RMID from a pool with a size that is platform