diff --git a/docs/slurm-exporter.md b/docs/slurm-exporter.md index c6c655c78..9fccd0668 100644 --- a/docs/slurm-exporter.md +++ b/docs/slurm-exporter.md @@ -94,7 +94,7 @@ Boolean state flag label convention: | Metric Name & Type | Description & Labels | |-------------------|---------------------| -| **slurm_node_info**
*Gauge* | Provides detailed information about SLURM nodes

**Labels:**
• `node_name` - Name of the SLURM node
• `instance_id` - Kubernetes instance identifier
• `state_base` - Base node state (IDLE, ALLOCATED, DOWN, ERROR, MIXED, UNKNOWN)
• `state_is_drain` - Whether node is in drain state ("true"/"false")
• `state_is_maintenance` - Whether node is in maintenance state ("true"/"false")
• `state_is_reserved` - Whether node is in reserved state ("true"/"false")
• `state_is_completing` - Whether node is in completing state ("true" or empty)
• `state_is_fail` - Whether node is in fail state ("true" or empty)
• `state_is_planned` - Whether node is in planned state ("true" or empty)
• `state_is_not_responding` - Whether the node is marked as not responding ("true" or empty)
• `state_is_invalid` - Whether the node state is considered invalid by SLURM ("true" or empty)
• `state_is_cloud` - Whether the node is a cloud node powered on/off dynamically via Slurm power saving ("true" or empty)
• `state_is_power_down` - Whether the node is pending power down ("true" or empty)
• `state_is_power_drain` - Whether the node is draining as part of power down ("true" or empty)
• `state_is_powered_down` - Whether the node is currently powered down ("true" or empty)
• `state_is_powering_down` - Whether the node is transitioning to powered down ("true" or empty)
• `state_is_powering_up` - Whether the node is transitioning to powered up ("true" or empty)
• `state_is_power_up` - Whether the node is pending power up ("true" or empty)
• `is_unavailable` - Computed by the exporter: "true" when the node is considered unavailable (DOWN+* or IDLE+DRAIN+*), empty string otherwise. Power-managed nodes (powering up/down or powered down) are never reported as unavailable
• `reservation_name` - Reservation that currently includes the node (trimmed to 50 characters)
• `address` - IP address of the node
• `reason` - Reason for current node state (empty string if node has no reason set)
• `comment` - Comment set on the node (e.g., by active checks when GPU health check fails) | +| **slurm_node_info**
*Gauge* | Provides detailed information about SLURM nodes

**Labels:**
• `node_name` - Name of the SLURM node
• `instance_id` - Kubernetes instance identifier
• `nodeset_name` - Name of the Soperator NodeSet from the worker Pod's `slurm.nebius.ai/nodeset` label, including for pending Pods; empty when the worker Pod is unavailable
• `state_base` - Base node state (IDLE, ALLOCATED, DOWN, ERROR, MIXED, UNKNOWN)
• `state_is_drain` - Whether node is in drain state ("true"/"false")
• `state_is_maintenance` - Whether node is in maintenance state ("true"/"false")
• `state_is_reserved` - Whether node is in reserved state ("true"/"false")
• `state_is_completing` - Whether node is in completing state ("true" or empty)
• `state_is_fail` - Whether node is in fail state ("true" or empty)
• `state_is_planned` - Whether node is in planned state ("true" or empty)
• `state_is_not_responding` - Whether the node is marked as not responding ("true" or empty)
• `state_is_invalid` - Whether the node state is considered invalid by SLURM ("true" or empty)
• `state_is_cloud` - Whether the node is a cloud node powered on/off dynamically via Slurm power saving ("true" or empty)
• `state_is_power_down` - Whether the node is pending power down ("true" or empty)
• `state_is_power_drain` - Whether the node is draining as part of power down ("true" or empty)
• `state_is_powered_down` - Whether the node is currently powered down ("true" or empty)
• `state_is_powering_down` - Whether the node is transitioning to powered down ("true" or empty)
• `state_is_powering_up` - Whether the node is transitioning to powered up ("true" or empty)
• `state_is_power_up` - Whether the node is pending power up ("true" or empty)
• `is_unavailable` - Computed by the exporter: "true" when the node is considered unavailable (DOWN+* or IDLE+DRAIN+*), empty string otherwise. Power-managed nodes (powering up/down or powered down) are never reported as unavailable
• `reservation_name` - Reservation that currently includes the node (trimmed to 50 characters)
• `address` - IP address of the node
• `reason` - Reason for current node state (empty string if node has no reason set)
• `comment` - Comment set on the node (e.g., by active checks when GPU health check fails) | | **slurm_node_gpu_seconds_total**
*Counter* | Total GPU seconds accumulated on SLURM nodes

**Labels:**
• `node_name` - Name of the SLURM node
• `state_base` - Base node state
• `state_is_drain` - Drain state flag
• `state_is_maintenance` - Maintenance state flag
• `state_is_reserved` - Reserved state flag | | **slurm_node_fails_total**
*Counter* | Total number of node state transitions to failed states (DOWN/DRAIN). Power-managed nodes (powering up/down or powered down) are excluded, since their DOWN/DRAIN states are part of the normal cloud lifecycle

**Labels:**
• `node_name` - Name of the SLURM node
• `state_base` - Base node state at time of failure
• `state_is_drain` - Drain state flag
• `state_is_maintenance` - Maintenance state flag
• `state_is_reserved` - Reserved state flag
• `reason` - Reason for the node failure | | **slurm_node_unavailability_duration_seconds**
*Histogram* | Duration of completed node unavailability events (DOWN+* or IDLE+DRAIN+*). Power-managed nodes (powering up/down or powered down) are excluded, since their DOWN/DRAIN states are part of the normal cloud lifecycle

**Labels:**
• `node_name` - Name of the SLURM node

**Note:** Observations are recorded when unavailability events complete. Duration tracking is reset on exporter restarts, which may affect accuracy | @@ -108,9 +108,9 @@ Boolean state flag label convention: | **slurm_node_memory_free_bytes**
*Gauge* | Free memory on the node in bytes

**Labels:**
• `node_name` - Name of the SLURM node | | **slurm_node_memory_effective_bytes**
*Gauge* | Effective memory on the node in bytes (total minus specialized memory reserved for system daemons)

**Labels:**
• `node_name` - Name of the SLURM node | | **slurm_node_partition**
*Gauge* | Maps nodes to their partitions, enabling partition-level aggregation via PromQL joins

**Labels:**
• `node_name` - Name of the SLURM node
• `partition` - Name of the SLURM partition | -| **slurm_node_nvlink_instance_group**
*Gauge* | Maps Slurm nodes to their NVLink instance group. Emitted only for nodes with the `topology.nebius.com/nvl-instance-group-id` Kubernetes label.

**Labels:**
• `node_name` - Name of the Slurm node and worker Pod
• `instance_id` - Compute instance identifier reported by Slurm
• `nvlink_instance_group` - Value of the Kubernetes Node's `topology.nebius.com/nvl-instance-group-id` label
• `nodeset_name` - Value of the Kubernetes Node's `slurm.nebius.ai/nodeset-name` label | +| **slurm_node_nvlink_instance_group**
*Gauge* | Maps Slurm nodes to their NVLink instance group. Emitted only for nodes with the `topology.nebius.com/nvl-instance-group-id` Kubernetes label.

**Labels:**
• `node_name` - Name of the Slurm node and worker Pod
• `instance_id` - Compute instance identifier reported by Slurm
• `nvlink_instance_group` - Value of the Kubernetes Node's `topology.nebius.com/nvl-instance-group-id` label
• `nodeset_name` - Name of the Soperator NodeSet from the worker Pod's `slurm.nebius.ai/nodeset` label | | **slurm_job_info**
*Gauge* | Detailed information about SLURM jobs

**Labels:**
• `job_id` - SLURM job identifier
• `job_state` - Current job state (PENDING, RUNNING, COMPLETED, FAILED, etc.)
• `job_state_reason` - Reason for current job state
• `slurm_partition` - SLURM partition name
• `job_name` - User-defined job name
• `user_name` - Username who submitted the job
• `user_id` - Numeric user ID who submitted the job
• `standard_error` - Path to stderr file
• `standard_output` - Path to stdout file
• `array_job_id` - Array job ID (if applicable)
• `array_task_id` - Array task identity. Either a single task index (e.g. `3`) for an exploded task or a range expression (e.g. `1-5`, `1,3,5-9:2`) for a collapsed array master record on the accounting path. Empty for non-array jobs.
• `submit_time` - When the job was submitted (Unix timestamp seconds, empty if not available or zero)
• `start_time` - When the job started execution (Unix timestamp seconds, empty if not available or zero)
• `end_time` - When the job completed (Unix timestamp seconds, empty if not available or zero). **Warning:** For non-terminal states like RUNNING, this may contain a future timestamp representing the forecasted end time based on the job's time limit
• `finished_time` - When the job actually finished for terminal states only (Unix timestamp seconds, empty for non-terminal states or if end_time is zero). Unlike `end_time`, this field only contains actual completion times, never forecasted values | -| **slurm_node_job**
*Gauge* | Mapping between jobs and the nodes they're running on

**Labels:**
• `job_id` - SLURM job identifier
• `node_name` - Name of the node running the job
• `nvlink_instance_group` - NVLink instance group of the node, or empty when unavailable
• `nodeset_name` - Slurm NodeSet name of the node, or empty when unavailable | +| **slurm_node_job**
*Gauge* | Mapping between jobs and the nodes they're running on

**Labels:**
• `job_id` - SLURM job identifier
• `node_name` - Name of the node running the job
• `nvlink_instance_group` - NVLink instance group of the node, or empty when unavailable
• `nodeset_name` - Name of the Soperator NodeSet from the worker Pod's `slurm.nebius.ai/nodeset` label, or empty when unavailable | | **slurm_job_duration_seconds**
*Gauge* | Job duration in seconds. For running jobs, this is the time elapsed since the job started. For completed jobs, this is the total execution time.

**Labels:**
• `job_id` - SLURM job identifier

**Notes:**
• Only exported for jobs with a valid start time
• For non-terminal states (RUNNING, etc.): duration = current_time - start_time
• For terminal states (COMPLETED, FAILED, etc.): duration = end_time - start_time (only if end_time is valid) | | **slurm_job_cpus**
*Gauge* | Number of CPUs allocated to the job

**Labels:**
• `job_id` - SLURM job identifier | | **slurm_job_memory_bytes**
*Gauge* | Memory allocated to the job in bytes

**Labels:**
• `job_id` - SLURM job identifier | diff --git a/helm/soperator-fluxcd/values.yaml b/helm/soperator-fluxcd/values.yaml index 5c203f0c2..f5f85cb1c 100644 --- a/helm/soperator-fluxcd/values.yaml +++ b/helm/soperator-fluxcd/values.yaml @@ -470,6 +470,8 @@ observability: requests: cpu: 100m grafana: + image: + tag: "11.6.14-security-04" nodeSelector: slurm.nebius.ai/nodeset: system affinity: diff --git a/helm/soperator-monitoring-dashboards/dashboards/cluster_health.json b/helm/soperator-monitoring-dashboards/dashboards/cluster_health.json index 356210f54..75b48aed4 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/cluster_health.json +++ b/helm/soperator-monitoring-dashboards/dashboards/cluster_health.json @@ -18,19 +18,38 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 22, + "id": 12, "links": [ { "asDropdown": false, "icon": "external link", "includeVars": false, "keepTime": true, - "tags": ["soperator", "gpu"], + "tags": [ + "soperator", + "gpu" + ], "targetBlank": true, "title": "GPU Stats", "tooltip": "", "type": "dashboards", "url": "" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "soperator", + "nodesets", + "overview" + ], + "targetBlank": true, + "title": "Nodesets Oveview", + "tooltip": "", + "type": "dashboards", + "url": "" } ], "panels": [ @@ -64,8 +83,7 @@ "mode": "absolute", "steps": [ { - "color": "text", - "value": null + "color": "text" } ] } @@ -87,7 +105,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -95,7 +115,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -104,7 +124,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "group(\n kube_customresource_slurmcluster_info{\n cr_version!=\"\"\n }\n) by(cr_version)\n", + "expr": "group(\n kube_customresource_slurmcluster_info{cluster=~\"$cluster\", \n cr_version!=\"\"\n }\n) by(cr_version)\n", "instant": true, "legendFormat": "{{version}}", "range": false, @@ -132,8 +152,7 @@ "mode": "absolute", "steps": [ { - "color": "super-light-blue", - "value": null + "color": "super-light-blue" } ] } @@ -155,7 +174,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -163,7 +184,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -172,7 +193,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count(\n cadvisor_version_info{}\n) by(iam_project_id)", + "expr": "count(\n cadvisor_version_info{cluster=~\"$cluster\"}\n) by(iam_project_id)", "instant": true, "legendFormat": "__auto", "range": false, @@ -200,8 +221,7 @@ "mode": "absolute", "steps": [ { - "color": "super-light-blue", - "value": null + "color": "super-light-blue" } ] } @@ -223,7 +243,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -231,7 +253,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -240,7 +262,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "group(\n kube_node_labels{label_nebius_com_gpu_name!=\"\"}\n) by (label_nebius_com_gpu_name)", + "expr": "group(\n kube_node_labels{cluster=~\"$cluster\", label_nebius_com_gpu_name!=\"\"}\n) by (label_nebius_com_gpu_name)", "instant": true, "legendFormat": "__auto", "range": false, @@ -252,68 +274,6 @@ "title": "GPU model", "type": "stat" }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "For GB platform only", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "super-light-purple", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.5.1", - "targets": [ - { - "editorMode": "code", - "exemplar": false, - "expr": "count(\n count by (label_slurm_nebius_ai_nodeset_name) (\n kube_node_labels{\n label_nebius_com_gpu_name=~\"GB.*\",\n label_slurm_nebius_ai_nodeset_name!=\"\"\n }\n )\n)", - "instant": true, - "range": false, - "refId": "A", - "useCustomValues": true, - "useDashValues": false - } - ], - "title": "Racks", - "type": "stat" - }, { "datasource": { "type": "prometheus", @@ -330,8 +290,7 @@ "mode": "absolute", "steps": [ { - "color": "super-light-purple", - "value": null + "color": "super-light-purple" } ] } @@ -341,7 +300,7 @@ "gridPos": { "h": 3, "w": 4, - "x": 20, + "x": 16, "y": 1 }, "id": 4, @@ -352,7 +311,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -360,12 +321,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "count(\n slurm_node_info{\n}\n) by()", + "expr": "count(\n slurm_node_info{cluster=~\"$cluster\"\n}\n) by()", "instant": true, "range": false, "refId": "A", @@ -393,8 +354,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -417,7 +377,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -425,7 +387,7 @@ "textMode": "value", "wideLayout": false }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -434,7 +396,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(\n tlast_change_over_time(\n (\n group(\n kube_customresource_slurmcluster_info{\n cr_version!=\"\"\n }\n ) by(cr_version)\n )[180d]\n )\n *\n 1000\n)\nand on(cr_version)\ngroup(\n kube_customresource_slurmcluster_info{\n cr_version!=\"\"\n }\n) by(cr_version)", + "expr": "(\n tlast_change_over_time(\n (\n group(\n kube_customresource_slurmcluster_info{cluster=~\"$cluster\", \n cr_version!=\"\"\n }\n ) by(cr_version)\n )[180d]\n )\n *\n 1000\n)\nand on(cr_version)\ngroup(\n kube_customresource_slurmcluster_info{cluster=~\"$cluster\", \n cr_version!=\"\"\n }\n) by(cr_version)", "instant": true, "legendFormat": "{{version}}", "range": false, @@ -462,8 +424,7 @@ "mode": "absolute", "steps": [ { - "color": "super-light-blue", - "value": null + "color": "super-light-blue" } ] } @@ -485,7 +446,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -493,7 +456,7 @@ "textMode": "name", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -502,7 +465,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count(\n cadvisor_version_info{}\n) by(mk8s_cluster_id)", + "expr": "count(\n cadvisor_version_info{cluster=~\"$cluster\"}\n) by(mk8s_cluster_id)", "instant": true, "legendFormat": "__auto", "range": false, @@ -530,8 +493,7 @@ "mode": "absolute", "steps": [ { - "color": "super-light-purple", - "value": null + "color": "super-light-purple" } ] } @@ -552,7 +514,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -560,12 +524,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "count(\n DCGM_FI_DEV_GPU_UTIL{\n \n }\n) by()", + "expr": "count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n \n }\n) by()", "instant": true, "range": false, "refId": "A", @@ -592,8 +556,7 @@ "mode": "absolute", "steps": [ { - "color": "super-light-purple", - "value": null + "color": "super-light-purple" } ] } @@ -614,7 +577,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -622,12 +587,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "count(\n count by (label_topology_nebius_com_nvl_instance_group_id) (\n kube_node_labels{\n label_nebius_com_gpu_name=~\"GB.*\",\n label_topology_nebius_com_nvl_instance_group_id!=\"\"\n }\n )\n)", + "expr": "count(\n count(slurm_node_nvlink_instance_group{cluster=~\"$cluster\"}) by (cluster, nvlink_instance_group)\n) by (cluster)", "instant": true, "range": false, "refId": "A", @@ -668,8 +633,7 @@ "mode": "absolute", "steps": [ { - "color": "light-orange", - "value": null + "color": "light-orange" } ] }, @@ -691,7 +655,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -699,12 +665,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "sum_over_time(\n count(\n DCGM_FI_DEV_GPU_UTIL{\n \n }\n >=\n 10\n)[1d:10m])\n/\nsum_over_time(\n count(\n DCGM_FI_DEV_GPU_UTIL{\n \n }\n)[1d:10m])\n* 100", + "expr": "sum_over_time(\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n \n }\n >=\n 10\n)[1d:10m])\n/\nsum_over_time(\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n \n }\n)[1d:10m])\n* 100", "instant": true, "legendFormat": "__auto", "range": false, @@ -734,8 +700,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -761,7 +726,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "fields": "", "values": false }, @@ -769,7 +736,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -778,7 +745,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg(\n (\n count(\n slurm_node_job{\n \n }\n ) by(job_id)\n * on(job_id)\n slurm_job_info{\n \n slurm_partition${partition_match_exp}\n }\n )\n >\n 1\n) by ()", + "expr": "avg(\n (\n count(\n slurm_node_job{cluster=~\"$cluster\"\n \n }\n ) by(job_id)\n * on(job_id)\n slurm_job_info{cluster=~\"$cluster\", \n \n slurm_partition${partition_match_exp}\n }\n )\n >\n 1\n) by ()", "hide": false, "instant": false, "legendFormat": "__auto", @@ -809,8 +776,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -832,7 +798,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "fields": "", "values": false }, @@ -840,7 +808,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -849,7 +817,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg(\n DCGM_FI_DEV_POWER_USAGE{\n \n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{\n \n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", + "expr": "avg(\n DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\"\n \n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n \n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", "hide": false, "instant": false, "range": true, @@ -877,8 +845,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -899,7 +866,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -907,12 +876,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "count(\n slurm_node_info{\n \n state_base=~\"IDLE|MIXED|ALLOCATED\",\n state_is_drain=\"false\"\n }\n) by ()\n", + "expr": "count(\n slurm_node_info{cluster=~\"$cluster\", \n \n state_base=~\"IDLE|MIXED|ALLOCATED\",\n state_is_drain=\"false\"\n }\n) by ()\n", "instant": true, "range": false, "refId": "A", @@ -940,8 +909,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -963,7 +931,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -971,12 +941,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "time() - kube_pod_created{\n \n namespace=~\"soperator\",\n pod=~\"controller-[0-9]+\"\n}", + "expr": "time() - kube_pod_created{cluster=~\"$cluster\", \n \n namespace=~\"soperator\",\n pod=~\"controller-[0-9]+\"\n}", "instant": true, "legendFormat": "{{pod}}", "range": false, @@ -1006,8 +976,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1032,7 +1001,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1040,7 +1011,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1048,7 +1019,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_avg(sum(increase(slurm_node_unavailability_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_avg(sum(increase(slurm_node_unavailability_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))", "hide": false, "range": true, "refId": "B", @@ -1076,8 +1047,7 @@ "mode": "absolute", "steps": [ { - "color": "light-orange", - "value": null + "color": "light-orange" } ] }, @@ -1099,7 +1069,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1107,12 +1079,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "sum_over_time(\n count(\n DCGM_FI_DEV_GPU_UTIL{\n \n }\n >=\n 10\n)[7d:1h])\n/\nsum_over_time(\n count(\n DCGM_FI_DEV_GPU_UTIL{\n }\n)[7d:1h])\n* 100", + "expr": "sum_over_time(\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n \n }\n >=\n 10\n)[7d:1h])\n/\nsum_over_time(\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n }\n)[7d:1h])\n* 100", "instant": true, "legendFormat": "__auto", "range": false, @@ -1142,8 +1114,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1169,7 +1140,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "fields": "", "values": false }, @@ -1177,7 +1150,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1186,7 +1159,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg(\n sum_over_time(\n (\n slurm_job_info{\n \n slurm_partition${partition_match_exp},\n job_state=\"RUNNING\"\n }\n )[1d:1m]\n )\n *\n 60\n) by ()", + "expr": "avg(\n sum_over_time(\n (\n slurm_job_info{cluster=~\"$cluster\", \n \n slurm_partition${partition_match_exp},\n job_state=\"RUNNING\"\n }\n )[1d:1m]\n )\n *\n 60\n) by ()", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1216,8 +1189,7 @@ "mode": "absolute", "steps": [ { - "color": "super-light-purple", - "value": null + "color": "super-light-purple" } ] }, @@ -1239,7 +1211,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1247,12 +1221,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {}\n)\navg(\n DCGM_FI_DEV_GPU_UTIL{filters}\n)", + "expr": "with (\n filters = {}\n)\navg(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", filters}\n)", "instant": false, "range": true, "refId": "A", @@ -1280,8 +1254,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "light-orange", @@ -1310,7 +1283,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1318,12 +1293,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "count(\n slurm_node_info{\n \n state_base=~\"DOWN|ERROR\",\n state_is_drain=\"true\"\n }\n) by ()\n", + "expr": "count(\n slurm_node_info{cluster=~\"$cluster\", \n \n state_base=~\"DOWN|ERROR\",\n state_is_drain=\"true\"\n }\n) by ()\n", "instant": true, "range": false, "refId": "A", @@ -1352,8 +1327,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "light-orange", @@ -1386,7 +1360,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1394,7 +1370,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1402,7 +1378,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(max by(node_name, reason, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (slurm_node_fails_total{state_is_maintenance=\"false\", })[$__range]))", + "expr": "sum(increase(max by(node_name, reason, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (slurm_node_fails_total{cluster=~\"$cluster\", state_is_maintenance=\"false\", })[$__range]))", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -1433,8 +1409,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1459,7 +1434,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1467,7 +1444,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1475,7 +1452,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_avg(sum(increase(slurm_node_draining_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_avg(sum(increase(slurm_node_draining_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))", "hide": false, "range": true, "refId": "B", @@ -1503,8 +1480,7 @@ "mode": "absolute", "steps": [ { - "color": "super-light-purple", - "value": null + "color": "super-light-purple" } ] }, @@ -1526,7 +1502,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1534,12 +1512,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {}\n)\ncount(\n slurm_job_info{\n filters,\n job_state=\"RUNNING\",\n slurm_partition${partition_match_exp},\n }\n)", + "expr": "with (\n filters = {}\n)\ncount(\n slurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=\"RUNNING\",\n slurm_partition${partition_match_exp},\n }\n)", "instant": true, "range": false, "refId": "A", @@ -1567,8 +1545,7 @@ "mode": "absolute", "steps": [ { - "color": "super-light-purple", - "value": null + "color": "super-light-purple" } ] } @@ -1589,7 +1566,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1597,12 +1576,12 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", "exemplar": false, - "expr": "count(\n slurm_node_info{state_base=\"ALLOCATED\"}\n) by(cluster)", + "expr": "count(\n slurm_node_info{cluster=~\"$cluster\", state_base=\"ALLOCATED\"}\n) by(cluster)", "instant": true, "range": false, "refId": "A", @@ -1629,8 +1608,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1655,7 +1633,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -1663,7 +1643,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1671,7 +1651,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum (\n increase(\n sum by (node_name, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (\n slurm_node_gpu_seconds_total{\n \n state_is_maintenance=\"false\",\n state_base=~\".*\"\n }\n )[7d]\n )\n )\n /\n (\n sum (\n increase(\n max by (node_name, reason, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (\n slurm_node_fails_total{\n \n state_is_maintenance=\"false\"\n }\n )[7d]\n )\n )\n or\n (\n sum (\n increase(\n slurm_node_gpu_seconds_total{\n \n state_is_maintenance=\"false\",\n state_base=~\".*\"\n }[7d]\n )\n ) * 0\n )\n )\n) / 3600", + "expr": "(\n sum (\n increase(\n sum by (node_name, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (\n slurm_node_gpu_seconds_total{cluster=~\"$cluster\", \n \n state_is_maintenance=\"false\",\n state_base=~\".*\"\n }\n )[7d]\n )\n )\n /\n (\n sum (\n increase(\n max by (node_name, reason, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (\n slurm_node_fails_total{cluster=~\"$cluster\", \n \n state_is_maintenance=\"false\"\n }\n )[7d]\n )\n )\n or\n (\n sum (\n increase(\n slurm_node_gpu_seconds_total{cluster=~\"$cluster\", \n \n state_is_maintenance=\"false\",\n state_base=~\".*\"\n }[7d]\n )\n ) * 0\n )\n )\n) / 3600", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -1749,8 +1729,7 @@ "mode": "absolute", "steps": [ { - "color": "light-orange", - "value": null + "color": "light-orange" } ] } @@ -1777,7 +1756,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1785,7 +1764,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n count(\n DCGM_FI_DEV_GPU_UTIL{\n }\n >=\n 10\n ) by(cluster)\n /\n count(\n DCGM_FI_DEV_GPU_UTIL{\n }\n ) by(cluster)\n)\n *\n100", + "expr": "(\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n }\n >=\n 10\n ) by(cluster)\n /\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n }\n ) by(cluster)\n)\n *\n100", "legendFormat": "__auto", "range": true, "refId": "A", @@ -1846,8 +1825,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1875,7 +1853,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1883,7 +1861,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum (\n increase(\n sum by (node_name, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (\n slurm_node_gpu_seconds_total{\n state_is_maintenance=\"false\",\n }\n )[7d]\n )\n )\n /\n (\n sum (\n increase(\n max by (node_name, reason, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (\n slurm_node_fails_total{\n state_is_maintenance=\"false\"\n }\n )[7d]\n )\n )\n or\n (\n sum (\n increase(\n slurm_node_gpu_seconds_total{\n state_is_maintenance=\"false\",\n }[7d]\n )\n ) * 0\n )\n )\n) / 36", + "expr": "(\n sum (\n increase(\n sum by (node_name, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (\n slurm_node_gpu_seconds_total{cluster=~\"$cluster\", \n state_is_maintenance=\"false\",\n }\n )[7d]\n )\n )\n /\n (\n sum (\n increase(\n max by (node_name, reason, state_base, state_is_drain, state_is_maintenance, state_is_reserved) (\n slurm_node_fails_total{cluster=~\"$cluster\", \n state_is_maintenance=\"false\"\n }\n )[7d]\n )\n )\n or\n (\n sum (\n increase(\n slurm_node_gpu_seconds_total{cluster=~\"$cluster\", \n state_is_maintenance=\"false\",\n }[7d]\n )\n ) * 0\n )\n )\n) / 36", "format": "time_series", "intervalFactor": 2, "legendFormat": "MTBF", @@ -1947,8 +1925,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1980,7 +1957,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -2031,7 +2011,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -2039,7 +2019,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(\n DCGM_FI_DEV_POWER_USAGE{\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", + "expr": "max(\n DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\"\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", "hide": false, "legendFormat": "max", "range": true, @@ -2053,7 +2033,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n DCGM_FI_DEV_POWER_USAGE{\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", + "expr": "avg(\n DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\"\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", "hide": false, "legendFormat": "avg", "range": true, @@ -2067,7 +2047,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(\n DCGM_FI_DEV_POWER_USAGE{\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", + "expr": "min(\n DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\"\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", "hide": false, "legendFormat": "min", "range": true, @@ -2131,8 +2111,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2161,11 +2140,11 @@ "sort": "none" } }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "editorMode": "code", - "expr": "sum(\n slurm_job_info{\n slurm_partition${partition_match_exp},\n job_state=\"PENDING\"\n }\n) by(job_state_reason)", + "expr": "sum(\n slurm_job_info{cluster=~\"$cluster\", \n slurm_partition${partition_match_exp},\n job_state=\"PENDING\"\n }\n) by(job_state_reason)", "legendFormat": "__auto", "range": true, "refId": "A", @@ -2229,8 +2208,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2262,7 +2240,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -2313,7 +2294,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.1", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -2321,7 +2302,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(\n DCGM_FI_PROF_SM_ACTIVE\n)", + "expr": "max(\n DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\"}\n)", "hide": false, "legendFormat": "max", "range": true, @@ -2335,7 +2316,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n DCGM_FI_PROF_SM_ACTIVE\n)", + "expr": "avg(\n DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\"}\n)", "hide": false, "legendFormat": "avg", "range": true, @@ -2349,7 +2330,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(\n DCGM_FI_PROF_SM_ACTIVE\n)", + "expr": "min(\n DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\"}\n)", "hide": false, "legendFormat": "min", "range": true, @@ -2369,302 +2350,6 @@ "x": 0, "y": 39 }, - "id": 103, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Proportion of GPUs with at least 10% utilization to total for selected racks", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "light-orange", - "mode": "shades" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "light-orange", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 104, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.5.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "(\n count(\n DCGM_FI_DEV_GPU_UTIL{\n slurm_nodeset_name=~\"$rack\"\n }\n >=\n 10\n ) by(cluster)\n /\n count(\n DCGM_FI_DEV_GPU_UTIL{\n slurm_nodeset_name=~\"$rack\"\n }\n ) by(cluster)\n)\n *\n100", - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useCustomValues": true, - "useDashValues": false - } - ], - "title": "Rack Utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "description": "Only for GPUs with at least 10% utilization for selected racks", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 50, - "gradientMode": "opacity", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "watt" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max" - }, - "properties": [ - { - "id": "custom.fillBelowTo", - "value": "min" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "avg" - }, - "properties": [ - { - "id": "custom.lineStyle", - "value": { - "dash": [10, 10], - "fill": "dash" - } - }, - { - "id": "color", - "value": { - "fixedColor": "super-light-green", - "mode": "fixed" - } - }, - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "min" - }, - "properties": [ - { - "id": "custom.fillOpacity", - "value": 0 - } - ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 105, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "11.5.1", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "max(\n DCGM_FI_DEV_POWER_USAGE{\n slurm_nodeset_name=~\"$rack\"\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{\n slurm_nodeset_name=~\"$rack\"\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", - "hide": false, - "legendFormat": "max", - "range": true, - "refId": "C", - "useCustomValues": true, - "useDashValues": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "avg(\n DCGM_FI_DEV_POWER_USAGE{\n slurm_nodeset_name=~\"$rack\"\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{\n slurm_nodeset_name=~\"$rack\"\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", - "hide": false, - "legendFormat": "avg", - "range": true, - "refId": "B", - "useCustomValues": true, - "useDashValues": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "editorMode": "code", - "expr": "min(\n DCGM_FI_DEV_POWER_USAGE{\n slurm_nodeset_name=~\"$rack\"\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{\n slurm_nodeset_name=~\"$rack\"\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", - "hide": false, - "legendFormat": "min", - "range": true, - "refId": "A", - "useCustomValues": true, - "useDashValues": false - } - ], - "title": "Cluster Power usage", - "type": "timeseries" - } - ], - "title": "Racks", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 40 - }, "id": 79, "panels": [ { @@ -2717,8 +2402,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2840,12 +2524,17 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 143 }, "id": 82, "options": { "legend": { - "calcs": ["mean", "lastNotNull", "max", "min"], + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -2864,7 +2553,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=~\"DOWN|ERROR\", state_is_maintenance!=\"true\", node_name=~\"$node_name\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=~\"DOWN|ERROR\", state_is_maintenance!=\"true\", node_name=~\"$node_name\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Down", @@ -2878,7 +2567,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=\"IDLE\", state_is_maintenance!=\"true\", state_is_drain=\"true\", node_name=~\"$node_name\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=\"IDLE\", state_is_maintenance!=\"true\", state_is_drain=\"true\", node_name=~\"$node_name\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Drained", @@ -2892,7 +2581,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=~\"ALLOCATED|MIXED\", state_is_maintenance!=\"true\", state_is_drain=\"true\", node_name=~\"$node_name\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=~\"ALLOCATED|MIXED\", state_is_maintenance!=\"true\", state_is_drain=\"true\", node_name=~\"$node_name\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Draining", @@ -2906,7 +2595,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=~\"ALLOCATED|MIXED\", state_is_maintenance!=\"true\", state_is_drain!=\"true\", node_name=~\"$node_name\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=~\"ALLOCATED|MIXED\", state_is_maintenance!=\"true\", state_is_drain!=\"true\", node_name=~\"$node_name\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Busy", @@ -2920,7 +2609,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=\"IDLE\", state_is_maintenance!=\"true\", state_is_drain!=\"true\", node_name=~\"$node_name\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=\"IDLE\", state_is_maintenance!=\"true\", state_is_drain!=\"true\", node_name=~\"$node_name\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Idle", @@ -2934,7 +2623,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_is_maintenance=\"true\", node_name=~\"$node_name\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_is_maintenance=\"true\", node_name=~\"$node_name\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Maintenance", @@ -2948,7 +2637,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=\"UNKNOWN\", state_is_maintenance!=\"true\", node_name=~\"$node_name\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=\"UNKNOWN\", state_is_maintenance!=\"true\", node_name=~\"$node_name\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Unknown", @@ -3013,8 +2702,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3026,7 +2714,7 @@ "h": 8, "w": 12, "x": 12, - "y": 48 + "y": 143 }, "id": 83, "options": { @@ -3053,7 +2741,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "with (\n node_fails = sum by (node_name) (\n # Down: Nodes in DOWN or ERROR state (not in maintenance)\n count by (node_name) (slurm_node_info{state_base=~\"DOWN|ERROR\", state_is_maintenance!=\"true\", node_name=~\"$node_name\", }) * 1 or\n # Drained: IDLE nodes with drain flag (not in maintenance)\n count by (node_name) (slurm_node_info{state_base=\"IDLE\", state_is_drain=\"true\", state_is_maintenance!=\"true\", node_name=~\"$node_name\", }) * 2 or\n # Draining: ALLOCATED/MIXED nodes with drain flag (not in maintenance)\n count by (node_name) (slurm_node_info{state_base=~\"ALLOCATED|MIXED\", state_is_drain=\"true\", state_is_maintenance!=\"true\", node_name=~\"$node_name\", }) * 3\n )\n)\nnode_fails\nor\n(\n # To show recovery as IDLE without drain state after failure\n count by (node_name) (slurm_node_info{state_base=~\"IDLE\", state_is_drain=\"false\", state_is_maintenance!=\"true\", node_name=~\"$node_name\", }) * 4\n and\n min_over_time(node_fails[15m:1m]) > 0\n)", + "expr": "with (\n node_fails = sum by (node_name) (\n # Down: Nodes in DOWN or ERROR state (not in maintenance)\n count by (node_name) (slurm_node_info{cluster=~\"$cluster\", state_base=~\"DOWN|ERROR\", state_is_maintenance!=\"true\", node_name=~\"$node_name\", }) * 1 or\n # Drained: IDLE nodes with drain flag (not in maintenance)\n count by (node_name) (slurm_node_info{cluster=~\"$cluster\", state_base=\"IDLE\", state_is_drain=\"true\", state_is_maintenance!=\"true\", node_name=~\"$node_name\", }) * 2 or\n # Draining: ALLOCATED/MIXED nodes with drain flag (not in maintenance)\n count by (node_name) (slurm_node_info{cluster=~\"$cluster\", state_base=~\"ALLOCATED|MIXED\", state_is_drain=\"true\", state_is_maintenance!=\"true\", node_name=~\"$node_name\", }) * 3\n )\n)\nnode_fails\nor\n(\n # To show recovery as IDLE without drain state after failure\n count by (node_name) (slurm_node_info{cluster=~\"$cluster\", state_base=~\"IDLE\", state_is_drain=\"false\", state_is_maintenance!=\"true\", node_name=~\"$node_name\", }) * 4\n and\n min_over_time(node_fails[15m:1m]) > 0\n)", "format": "time_series", "intervalFactor": 2, "legendFormat": "__auto", @@ -3131,8 +2819,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3172,7 +2859,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -3191,7 +2881,7 @@ "h": 7, "w": 12, "x": 0, - "y": 141 + "y": 236 }, "id": 31, "options": { @@ -3215,7 +2905,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(1, sum(increase(slurm_node_unavailability_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(1, sum(increase(slurm_node_unavailability_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "max", "range": true, @@ -3229,7 +2919,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_avg(sum(increase(slurm_node_unavailability_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_avg(sum(increase(slurm_node_unavailability_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "avg", "range": true, @@ -3243,7 +2933,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0, sum(increase(slurm_node_unavailability_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0, sum(increase(slurm_node_unavailability_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "min", "range": true, @@ -3307,8 +2997,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3348,7 +3037,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -3367,7 +3059,7 @@ "h": 7, "w": 12, "x": 12, - "y": 141 + "y": 236 }, "id": 73, "options": { @@ -3391,7 +3083,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(1, sum(increase(slurm_node_draining_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(1, sum(increase(slurm_node_draining_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "max", "range": true, @@ -3405,7 +3097,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_avg(sum(increase(slurm_node_draining_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_avg(sum(increase(slurm_node_draining_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "avg", "range": true, @@ -3419,7 +3111,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0, sum(increase(slurm_node_draining_duration_seconds_bucket{}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0, sum(increase(slurm_node_draining_duration_seconds_bucket{cluster=~\"$cluster\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "min", "range": true, @@ -3456,8 +3148,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3615,7 +3306,7 @@ "h": 8, "w": 24, "x": 0, - "y": 148 + "y": 243 }, "id": 84, "options": { @@ -3624,7 +3315,9 @@ "countRows": false, "enablePagination": true, "fields": "", - "reducer": ["sum"], + "reducer": [ + "sum" + ], "show": false }, "frameIndex": 0, @@ -3640,7 +3333,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {},\n running_jobs = group(slurm_job_info{filters, job_state=~\"RUNNING\"}) by (job_id, job_name),\n node_fails = (\n # Get nodes with problems: DOWN/ERROR states or drain flag\n slurm_node_info{filters, state_base=~\"DOWN|ERROR\", state_is_maintenance!=\"true\", node_name=~\"$node_name\"} or\n slurm_node_info{filters, state_is_drain=\"true\", state_is_maintenance!=\"true\", node_name=~\"$node_name\"}\n ),\n node_fails_with_job_id = node_fails * on (node_name) group_left(job_id) slurm_node_job{filters}\n)\n(\n node_fails_with_job_id\n and on (job_id)\n (\n running_jobs offset -60m or running_jobs offset 60m\n )\n)\nor\nnode_fails", + "expr": "with (\n filters = {},\n running_jobs = group(slurm_job_info{cluster=~\"$cluster\", filters, job_state=~\"RUNNING\"}) by (job_id, job_name),\n node_fails = (\n # Get nodes with problems: DOWN/ERROR states or drain flag\n slurm_node_info{cluster=~\"$cluster\", filters, state_base=~\"DOWN|ERROR\", state_is_maintenance!=\"true\", node_name=~\"$node_name\"} or\n slurm_node_info{cluster=~\"$cluster\", filters, state_is_drain=\"true\", state_is_maintenance!=\"true\", node_name=~\"$node_name\"}\n ),\n node_fails_with_job_id = node_fails * on (node_name) group_left(job_id) slurm_node_job{cluster=~\"$cluster\", filters}\n)\n(\n node_fails_with_job_id\n and on (job_id)\n (\n running_jobs offset -60m or running_jobs offset 60m\n )\n)\nor\nnode_fails", "format": "table", "instant": false, "legendFormat": "__auto", @@ -3672,19 +3365,29 @@ "options": { "fields": { "Time": { - "aggregations": ["first", "last"], + "aggregations": [ + "first", + "last" + ], "operation": "aggregate" }, "Time (first)": { - "aggregations": ["first", "last"], + "aggregations": [ + "first", + "last" + ], "operation": "aggregate" }, "fail_reason": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "fail_reason (first)": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "instance_id": { @@ -3692,7 +3395,9 @@ "operation": "groupby" }, "job_id": { - "aggregations": ["uniqueValues"], + "aggregations": [ + "uniqueValues" + ], "operation": "aggregate" }, "node_name": { @@ -3700,7 +3405,9 @@ "operation": "groupby" }, "reason": { - "aggregations": ["uniqueValues"], + "aggregations": [ + "uniqueValues" + ], "operation": "aggregate" }, "state_base": { @@ -3735,7 +3442,9 @@ }, "mode": "reduceRow", "reduce": { - "include": ["job_id (uniqueValues)"], + "include": [ + "job_id (uniqueValues)" + ], "reducer": "uniqueValues" } } @@ -3783,17 +3492,63 @@ "destinationType": "string", "enumConfig": { "text": [ - [["37166"]], - [["37176"]], - [["37174"]], - [["37175"]], - [["37173"]], - [["37226", "37232"]], - [["30590"]], - [["37228", "37288"]], - [["37285"]], - [["37286"]], - [["37359"]] + [ + [ + "37166" + ] + ], + [ + [ + "37176" + ] + ], + [ + [ + "37174" + ] + ], + [ + [ + "37175" + ] + ], + [ + [ + "37173" + ] + ], + [ + [ + "37226", + "37232" + ] + ], + [ + [ + "30590" + ] + ], + [ + [ + "37228", + "37288" + ] + ], + [ + [ + "37285" + ] + ], + [ + [ + "37286" + ] + ], + [ + [ + "37359" + ] + ] ] }, "joinWith": "&var-slurm_job=", @@ -3816,7 +3571,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 40 }, "id": 80, "panels": [ @@ -3846,8 +3601,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -3969,8 +3723,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "#EAB839", @@ -4023,8 +3776,7 @@ "mode": "absolute", "steps": [ { - "color": "red", - "value": null + "color": "red" }, { "color": "orange", @@ -4087,8 +3839,7 @@ "mode": "absolute", "steps": [ { - "color": "light-blue", - "value": null + "color": "light-blue" } ] } @@ -4101,7 +3852,7 @@ "h": 12, "w": 24, "x": 0, - "y": 49 + "y": 144 }, "id": 92, "options": { @@ -4110,7 +3861,9 @@ "countRows": false, "enablePagination": true, "fields": "", - "reducer": ["sum"], + "reducer": [ + "sum" + ], "show": false }, "frameIndex": 0, @@ -4126,7 +3879,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {}\n)\n# Main job info\nslurm_job_info{\n filters,\n job_state=~\"RUNNING\", \n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}\n* on (job_id) group_left()\ncount by (job_id) (slurm_node_job{filters})", + "expr": "with (\n filters = {}\n)\n# Main job info\nslurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=~\"RUNNING\", \n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}\n* on (job_id) group_left()\ncount by (job_id) (slurm_node_job{cluster=~\"$cluster\", filters})", "format": "table", "instant": true, "range": false, @@ -4141,7 +3894,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {}\n)\n# Job duration\nmax by (job_id) (slurm_job_duration_seconds{filters})\n* on (job_id) group_left()\nslurm_job_info{\n filters,\n job_state=~\"RUNNING\", \n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}", + "expr": "with (\n filters = {}\n)\n# Job duration\nmax by (job_id) (slurm_job_duration_seconds{cluster=~\"$cluster\", filters})\n* on (job_id) group_left()\nslurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=~\"RUNNING\", \n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}", "format": "table", "hide": false, "instant": true, @@ -4156,7 +3909,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {}\n)\n# GPU Util\navg(\n label_move(\n DCGM_FI_DEV_GPU_UTIL{filters}\n , 'hpc_job','job_id')\n) by (job_id)\nand on (job_id)\nslurm_job_info{\n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}", + "expr": "with (\n filters = {}\n)\n# GPU Util\navg(\n label_move(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", filters}\n , 'hpc_job','job_id')\n) by (job_id)\nand on (job_id)\nslurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}", "format": "table", "hide": false, "instant": true, @@ -4171,7 +3924,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {}\n)\n# SM Util\navg(\n label_move(\n DCGM_FI_PROF_SM_ACTIVE{filters}\n , 'hpc_job','job_id')\n) by (job_id)\nand on (job_id)\nslurm_job_info{\n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}", + "expr": "with (\n filters = {}\n)\n# SM Util\navg(\n label_move(\n DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\", filters}\n , 'hpc_job','job_id')\n) by (job_id)\nand on (job_id)\nslurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}", "format": "table", "hide": false, "instant": true, @@ -4194,30 +3947,45 @@ "options": { "fields": { "Time": { - "aggregations": ["first", "last"] + "aggregations": [ + "first", + "last" + ] }, "Value": { - "aggregations": ["lastNotNull"], + "aggregations": [ + "lastNotNull" + ], "operation": "aggregate" }, "Value #A": { - "aggregations": ["lastNotNull"], + "aggregations": [ + "lastNotNull" + ], "operation": "aggregate" }, "Value #B": { - "aggregations": ["lastNotNull"], + "aggregations": [ + "lastNotNull" + ], "operation": "aggregate" }, "Value #C": { - "aggregations": ["mean"], + "aggregations": [ + "mean" + ], "operation": "aggregate" }, "Value #D": { - "aggregations": ["mean"], + "aggregations": [ + "mean" + ], "operation": "aggregate" }, "finished_time": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "job_id": { @@ -4225,31 +3993,47 @@ "operation": "groupby" }, "job_name": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "job_state": { - "aggregations": ["first"] + "aggregations": [ + "first" + ] }, "job_state_reason": { - "aggregations": ["first"] + "aggregations": [ + "first" + ] }, "slurm_job_info": { - "aggregations": ["last"], + "aggregations": [ + "last" + ], "operation": "aggregate" }, "slurm_partition": { - "aggregations": ["firstNotNull"], + "aggregations": [ + "firstNotNull" + ], "operation": "aggregate" }, "start_time": { - "aggregations": ["first"] + "aggregations": [ + "first" + ] }, "submit_time": { - "aggregations": ["first"] + "aggregations": [ + "first" + ] }, "user_name": { - "aggregations": ["firstNotNull"], + "aggregations": [ + "firstNotNull" + ], "operation": "aggregate" } }, @@ -4372,8 +4156,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4389,7 +4172,7 @@ "h": 7, "w": 12, "x": 0, - "y": 61 + "y": 156 }, "id": 19, "options": { @@ -4413,7 +4196,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum(\n slurm_job_info{\n \n slurm_partition${partition_match_exp},\n job_state=~\"COMPLETED\"\n }\n and on(job_id)\n (\n count(\n slurm_node_job{ }\n ) by(job_id)\n >=\n 0\n )\n ) by (cluster)\n /\n sum(\n slurm_job_info{\n slurm_partition${partition_match_exp},\n job_state=~\"COMPLETED|FAILED|CANCELLED|TIMEOUT|DEADLINE\",\n }\n and on(job_id)\n (\n count(\n slurm_node_job{ }\n ) by(job_id)\n >=\n 0\n )\n ) by (cluster)\n) \n *\n100", + "expr": "(\n sum(\n slurm_job_info{cluster=~\"$cluster\", \n \n slurm_partition${partition_match_exp},\n job_state=~\"COMPLETED\"\n }\n and on(job_id)\n (\n count(\n slurm_node_job{cluster=~\"$cluster\" }\n ) by(job_id)\n >=\n 0\n )\n ) by (cluster)\n /\n sum(\n slurm_job_info{cluster=~\"$cluster\", \n slurm_partition${partition_match_exp},\n job_state=~\"COMPLETED|FAILED|CANCELLED|TIMEOUT|DEADLINE\",\n }\n and on(job_id)\n (\n count(\n slurm_node_job{cluster=~\"$cluster\" }\n ) by(job_id)\n >=\n 0\n )\n ) by (cluster)\n) \n *\n100", "hide": false, "legendFormat": "__auto", "range": true, @@ -4476,8 +4259,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -4505,7 +4287,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -4540,7 +4325,7 @@ "h": 7, "w": 12, "x": 12, - "y": 61 + "y": 156 }, "id": 10, "options": { @@ -4564,7 +4349,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(\n (\n count(\n slurm_node_job{\n }\n ) by(job_id)\n * on(job_id)\n slurm_job_info{\n \n slurm_partition${partition_match_exp}\n }\n )\n >\n 0\n) by()", + "expr": "max(\n (\n count(\n slurm_node_job{cluster=~\"$cluster\"\n }\n ) by(job_id)\n * on(job_id)\n slurm_job_info{cluster=~\"$cluster\", \n \n slurm_partition${partition_match_exp}\n }\n )\n >\n 0\n) by()", "hide": false, "legendFormat": "max", "range": true, @@ -4578,7 +4363,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n (\n count(\n slurm_node_job{\n }\n ) by(job_id)\n * on(job_id)\n slurm_job_info{\n \n slurm_partition${partition_match_exp}\n }\n )\n >\n 0\n) by()", + "expr": "avg(\n (\n count(\n slurm_node_job{cluster=~\"$cluster\"\n }\n ) by(job_id)\n * on(job_id)\n slurm_job_info{cluster=~\"$cluster\", \n \n slurm_partition${partition_match_exp}\n }\n )\n >\n 0\n) by()", "hide": false, "legendFormat": "avg", "range": true, @@ -4592,7 +4377,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(\n (\n count(\n slurm_node_job{\n }\n ) by(job_id)\n * on(job_id)\n slurm_job_info{\n \n slurm_partition${partition_match_exp}\n }\n )\n >\n 0\n) by()", + "expr": "min(\n (\n count(\n slurm_node_job{cluster=~\"$cluster\"\n }\n ) by(job_id)\n * on(job_id)\n slurm_job_info{cluster=~\"$cluster\", \n \n slurm_partition${partition_match_exp}\n }\n )\n >\n 0\n) by()", "hide": false, "legendFormat": "min", "range": true, @@ -4655,8 +4440,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -4684,7 +4468,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -4719,7 +4506,7 @@ "h": 7, "w": 12, "x": 0, - "y": 68 + "y": 163 }, "id": 74, "options": { @@ -4743,7 +4530,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(\n slurm_job_duration_seconds\n and on(job_id)\n slurm_job_info{slurm_partition${partition_match_exp} }\n)", + "expr": "max(\n slurm_job_duration_seconds{cluster=~\"$cluster\"}\n and on(job_id)\n slurm_job_info{cluster=~\"$cluster\", slurm_partition${partition_match_exp} }\n)", "hide": false, "legendFormat": "max", "range": true, @@ -4757,7 +4544,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n slurm_job_duration_seconds{ }\n and on(job_id)\n slurm_job_info{slurm_partition${partition_match_exp} }\n)", + "expr": "avg(\n slurm_job_duration_seconds{cluster=~\"$cluster\" }\n and on(job_id)\n slurm_job_info{cluster=~\"$cluster\", slurm_partition${partition_match_exp} }\n)", "hide": false, "legendFormat": "avg", "range": true, @@ -4771,7 +4558,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(\n slurm_job_duration_seconds{ }\n and on(job_id)\n slurm_job_info{slurm_partition${partition_match_exp} }\n)", + "expr": "min(\n slurm_job_duration_seconds{cluster=~\"$cluster\" }\n and on(job_id)\n slurm_job_info{cluster=~\"$cluster\", slurm_partition${partition_match_exp} }\n)", "hide": false, "legendFormat": "min", "range": true, @@ -4813,8 +4600,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4829,7 +4615,7 @@ "h": 8, "w": 12, "x": 12, - "y": 68 + "y": 163 }, "id": 77, "options": { @@ -4852,7 +4638,7 @@ { "editorMode": "code", "exemplar": false, - "expr": "histogram(\n count(slurm_node_job) by (job_id)\n and on(job_id)\n slurm_job_info{\n slurm_partition${partition_match_exp}\n }\n)", + "expr": "histogram(\n count(slurm_node_job{cluster=~\"$cluster\"}) by (job_id)\n and on(job_id)\n slurm_job_info{cluster=~\"$cluster\", \n slurm_partition${partition_match_exp}\n }\n)", "format": "time_series", "hide": false, "instant": false, @@ -4907,8 +4693,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -4920,7 +4705,7 @@ "h": 7, "w": 12, "x": 0, - "y": 75 + "y": 170 }, "id": 72, "options": { @@ -4949,7 +4734,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "floor(\n avg_over_time(\n (\n (\n (\n round((count(DCGM_FI_DEV_FB_USED{hpc_job!=\"\"} > 2) by(hpc_job) / 8) > 2)\n -\n round(\n (count(DCGM_FI_DEV_POWER_USAGE{hpc_job!=\"\"} > 120) by(hpc_job) / 8)\n >\n 2\n )\n )\n /\n round((count(DCGM_FI_DEV_FB_USED{hpc_job!=\"\", } > 2) by(hpc_job) / 8) > 2)\n )\n *\n 100\n )[1h:1m]\n) >= 70\n)\n and on(hpc_job)\nlabel_move(slurm_job_info{job_state=\"RUNNING\"}, 'job_id', 'hpc_job')\nand on (hpc_job)\ncount_over_time( #filter out short durations\navg_over_time((\n (\n (\n (\n round((count(DCGM_FI_DEV_FB_USED{hpc_job!=\"\"} > 2) by(hpc_job) / 8) > 2)\n -\n round(\n (count(DCGM_FI_DEV_POWER_USAGE{hpc_job!=\"\"} > 120) by(hpc_job) / 8)\n >\n 2\n )\n )\n /\n round((count(DCGM_FI_DEV_FB_USED{hpc_job!=\"\"} > 2) by(hpc_job) / 8) > 2)\n )\n *\n 100\n )[1h:1m]\n) > 70\n and on(hpc_job)\nlabel_move(slurm_job_info{job_state=\"RUNNING\"}, 'job_id', 'hpc_job')\n)[1h:1m]\n) > 20", + "expr": "floor(\n avg_over_time(\n (\n (\n (\n round((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", hpc_job!=\"\"} > 2) by(hpc_job) / 8) > 2)\n -\n round(\n (count(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", hpc_job!=\"\"} > 120) by(hpc_job) / 8)\n >\n 2\n )\n )\n /\n round((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", hpc_job!=\"\", } > 2) by(hpc_job) / 8) > 2)\n )\n *\n 100\n )[1h:1m]\n) >= 70\n)\n and on(hpc_job)\nlabel_move(slurm_job_info{cluster=~\"$cluster\", job_state=\"RUNNING\"}, 'job_id', 'hpc_job')\nand on (hpc_job)\ncount_over_time( #filter out short durations\navg_over_time((\n (\n (\n (\n round((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", hpc_job!=\"\"} > 2) by(hpc_job) / 8) > 2)\n -\n round(\n (count(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", hpc_job!=\"\"} > 120) by(hpc_job) / 8)\n >\n 2\n )\n )\n /\n round((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", hpc_job!=\"\"} > 2) by(hpc_job) / 8) > 2)\n )\n *\n 100\n )[1h:1m]\n) > 70\n and on(hpc_job)\nlabel_move(slurm_job_info{cluster=~\"$cluster\", job_state=\"RUNNING\"}, 'job_id', 'hpc_job')\n)[1h:1m]\n) > 20", "format": "time_series", "fullMetaSearch": false, "hide": false, @@ -5104,8 +4889,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5120,7 +4904,7 @@ "h": 22, "w": 12, "x": 12, - "y": 76 + "y": 171 }, "id": 90, "options": { @@ -5148,7 +4932,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "with (\n filters = {slurm_partition${partition_match_exp}},\n)\nsum by (job_id,)(\n count by (job_id, job_name) (slurm_job_info{job_state=\"PENDING\", filters}) * 1 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"CONFIGURING\",filters}) * 2 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"RUNNING\", filters}) * 3 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"COMPLETING\", filters}) * 4 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"COMPLETED\", filters}) * 5 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"CANCELLED\", filters}) * 6 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"FAILED\", filters}) * 7 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"TIMEOUT\", filters}) * 8 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"PREEMPTED\", filters}) * 9 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"NODE_FAIL\", filters}) * 10 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"BOOT_FAIL\", filters}) * 11 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"DEADLINE\", filters}) * 12 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"OUT_OF_MEMORY\", filters}) * 13 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"REQUEUED\", filters}) * 14 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"RESIZING\", filters}) * 15 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"SUSPENDED\", filters}) * 16 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"STOPPED\", filters}) * 17 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"POWER_UP_NODE\", filters}) * 18 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"SIGNALING\", filters}) * 19 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"STAGE_OUT\", filters}) * 20 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"UPDATE_DB\", filters}) * 21\n)", + "expr": "with (\n filters = {slurm_partition${partition_match_exp}},\n)\nsum by (job_id,)(\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"PENDING\", filters}) * 1 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"CONFIGURING\",filters}) * 2 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"RUNNING\", filters}) * 3 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"COMPLETING\", filters}) * 4 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"COMPLETED\", filters}) * 5 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"CANCELLED\", filters}) * 6 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"FAILED\", filters}) * 7 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"TIMEOUT\", filters}) * 8 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"PREEMPTED\", filters}) * 9 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"NODE_FAIL\", filters}) * 10 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"BOOT_FAIL\", filters}) * 11 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"DEADLINE\", filters}) * 12 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"OUT_OF_MEMORY\", filters}) * 13 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"REQUEUED\", filters}) * 14 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"RESIZING\", filters}) * 15 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"SUSPENDED\", filters}) * 16 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"STOPPED\", filters}) * 17 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"POWER_UP_NODE\", filters}) * 18 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"SIGNALING\", filters}) * 19 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"STAGE_OUT\", filters}) * 20 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"UPDATE_DB\", filters}) * 21\n)", "format": "time_series", "hide": false, "legendFormat": "{{job_id}}", @@ -5210,8 +4994,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5227,12 +5010,17 @@ "h": 8, "w": 12, "x": 0, - "y": 82 + "y": 177 }, "id": 87, "options": { "legend": { - "calcs": ["mean", "lastNotNull", "max", "min"], + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5251,7 +5039,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count by(job_state) (slurm_job_info{user_name=~\"$user_name\", slurm_partition${partition_match_exp}, })", + "expr": "count by(job_state) (slurm_job_info{cluster=~\"$cluster\", user_name=~\"$user_name\", slurm_partition${partition_match_exp}, })", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{job_state}}", @@ -5313,8 +5101,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5330,12 +5117,17 @@ "h": 8, "w": 12, "x": 0, - "y": 90 + "y": 185 }, "id": 88, "options": { "legend": { - "calcs": ["mean", "lastNotNull", "max", "min"], + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], "displayMode": "table", "placement": "bottom", "showLegend": true @@ -5354,7 +5146,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count by(slurm_partition) (slurm_job_info{job_state=\"RUNNING\", user_name=~\"$user_name\", slurm_partition${partition_match_exp}, })", + "expr": "count by(slurm_partition) (slurm_job_info{cluster=~\"$cluster\", job_state=\"RUNNING\", user_name=~\"$user_name\", slurm_partition${partition_match_exp}, })", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{slurm_partition}}", @@ -5392,8 +5184,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -5590,7 +5381,7 @@ "h": 12, "w": 24, "x": 0, - "y": 98 + "y": 193 }, "id": 91, "options": { @@ -5599,7 +5390,9 @@ "countRows": false, "enablePagination": true, "fields": "", - "reducer": ["sum"], + "reducer": [ + "sum" + ], "show": false }, "frameIndex": 0, @@ -5614,7 +5407,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "with (\n filters = {}\n)\nslurm_job_info{\n filters,\n job_state=~\"COMPLETED|CANCELLED|FAILED|TIMEOUT\", \n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}\n* on (job_id) group_left()\ncount by (job_id) (slurm_node_job{filters}) ", + "expr": "with (\n filters = {}\n)\nslurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=~\"COMPLETED|CANCELLED|FAILED|TIMEOUT\", \n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}, \n}\n* on (job_id) group_left()\ncount by (job_id) (slurm_node_job{cluster=~\"$cluster\", filters}) ", "format": "table", "instant": false, "range": true, @@ -5648,7 +5441,10 @@ "options": { "fields": { "Time": { - "aggregations": ["first", "last"], + "aggregations": [ + "first", + "last" + ], "operation": "aggregate" }, "Value": { @@ -5656,7 +5452,9 @@ "operation": "groupby" }, "finished_time": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "job_id": { @@ -5664,35 +5462,51 @@ "operation": "groupby" }, "job_name": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "job_state": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "job_state_reason": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "slurm_job_info": { - "aggregations": ["last"], + "aggregations": [ + "last" + ], "operation": "aggregate" }, "slurm_partition": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "start_time": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "submit_time": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" }, "user_name": { - "aggregations": ["first"], + "aggregations": [ + "first" + ], "operation": "aggregate" } }, @@ -5829,7 +5643,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 41 }, "id": 93, "panels": [ @@ -5882,8 +5696,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5899,12 +5712,16 @@ "h": 8, "w": 12, "x": 0, - "y": 50 + "y": 145 }, "id": 85, "options": { "legend": { - "calcs": ["lastNotNull", "max", "min"], + "calcs": [ + "lastNotNull", + "max", + "min" + ], "displayMode": "table", "placement": "right", "showLegend": true @@ -5923,7 +5740,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count by(user_name) (slurm_job_info{job_state=\"RUNNING\", user_name=~\"$user_name\", slurm_partition${partition_match_exp}, })", + "expr": "count by(user_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"RUNNING\", user_name=~\"$user_name\", slurm_partition${partition_match_exp}, })", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{user_name}}", @@ -5985,8 +5802,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6002,12 +5818,16 @@ "h": 8, "w": 12, "x": 12, - "y": 50 + "y": 145 }, "id": 86, "options": { "legend": { - "calcs": ["lastNotNull", "max", "min"], + "calcs": [ + "lastNotNull", + "max", + "min" + ], "displayMode": "table", "placement": "right", "showLegend": true @@ -6026,7 +5846,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count by(user_name) (slurm_job_info{user_name=~\"$user_name\", slurm_partition${partition_match_exp}, })", + "expr": "count by(user_name) (slurm_job_info{cluster=~\"$cluster\", user_name=~\"$user_name\", slurm_partition${partition_match_exp}, })", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{user_name}}", @@ -6089,8 +5909,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -6102,12 +5921,15 @@ "h": 8, "w": 12, "x": 0, - "y": 58 + "y": 153 }, "id": 94, "options": { "legend": { - "calcs": ["max", "min"], + "calcs": [ + "max", + "min" + ], "displayMode": "table", "placement": "right", "showLegend": true @@ -6126,7 +5948,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "with (\n filters = {}\n)\navg(\n label_move(DCGM_FI_DEV_GPU_UTIL{filters, hpc_job!=\"\"}, 'hpc_job','job_id')\n * on (job_id) group_left(user_name)\n slurm_job_info{\n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}\n }\n) by (user_name)", + "expr": "with (\n filters = {}\n)\navg(\n label_move(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", filters, hpc_job!=\"\"}, 'hpc_job','job_id')\n * on (job_id) group_left(user_name)\n slurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}\n }\n) by (user_name)", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -6190,8 +6012,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -6233,7 +6054,7 @@ "h": 8, "w": 12, "x": 12, - "y": 58 + "y": 153 }, "id": 75, "options": { @@ -6253,7 +6074,7 @@ "targets": [ { "editorMode": "code", - "expr": "count(\n count(\n slurm_job_info{\n \n slurm_partition${partition_match_exp}\n }\n ) by (user_name)\n) by ()", + "expr": "count(\n count(\n slurm_job_info{cluster=~\"$cluster\", \n \n slurm_partition${partition_match_exp}\n }\n ) by (user_name)\n) by ()", "legendFormat": "count", "range": true, "refId": "A", @@ -6326,12 +6147,15 @@ "h": 8, "w": 12, "x": 0, - "y": 66 + "y": 161 }, "id": 95, "options": { "legend": { - "calcs": ["max", "min"], + "calcs": [ + "max", + "min" + ], "displayMode": "table", "placement": "right", "showLegend": true @@ -6350,7 +6174,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "with (\n filters = {}\n)\ncount(\n slurm_node_job{filters}\n * on (job_id) group_left(user_name)\n slurm_job_info{\n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}\n }\n) by (user_name)", + "expr": "with (\n filters = {}\n)\ncount(\n slurm_node_job{cluster=~\"$cluster\", filters}\n * on (job_id) group_left(user_name)\n slurm_job_info{cluster=~\"$cluster\", \n filters,\n job_state=\"RUNNING\",\n user_name=~\"$user_name\", \n slurm_partition${partition_match_exp}\n }\n) by (user_name)", "format": "time_series", "intervalFactor": 1, "legendFormat": "__auto", @@ -6369,48 +6193,91 @@ } ], "preload": false, - "schemaVersion": 40, - "tags": ["nebius", "soperator", "overview", "cluster"], + "schemaVersion": 41, + "tags": [ + "nebius", + "soperator", + "overview", + "cluster" + ], "templating": { "list": [ { "current": { - "selected": true, "text": "VictoriaMetrics", "value": "VictoriaMetrics" }, "hide": 2, "includeAll": false, "label": "Datasource", - "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", "allowCustomValue": false, "current": { - "text": ["All"], - "value": ["$__all"] + "text": "All", + "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(kube_node_labels{label_nebius_com_gpu_name=~\"GB.*\"},label_slurm_nebius_ai_nodeset_name)", - "description": "For GB platforms", + "definition": "label_values(slurm_node_info{cluster=~\"$cluster\"},nodeset_name)", + "description": "", "includeAll": true, - "label": "Rack", + "label": "Nodeset", "multi": true, - "name": "rack", + "name": "nodeset", "options": [], "query": { "qryType": 1, - "query": "label_values(kube_node_labels{label_nebius_com_gpu_name=~\"GB.*\"},label_slurm_nebius_ai_nodeset_name)", + "query": "label_values(slurm_node_info{cluster=~\"$cluster\"},nodeset_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -6418,16 +6285,17 @@ "type": "query" }, { + "allValue": ".*", "allowCustomValue": false, "current": { "text": "All", - "value": ["$__all"] + "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(slurm_node_info{node_name=~\"$rack-.*\"}, node_name)", + "definition": "label_values(slurm_node_info{cluster=~\"$cluster\", nodeset_name=~\"$nodeset\"}, node_name)", "description": "", "includeAll": true, "label": "Worker Node", @@ -6436,7 +6304,7 @@ "options": [], "query": { "qryType": 5, - "query": "label_values(slurm_node_info{node_name=~\"$rack-.*\"}, node_name)", + "query": "label_values(slurm_node_info{cluster=~\"$cluster\", nodeset_name=~\"$nodeset\"}, node_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -6449,13 +6317,15 @@ "allowCustomValue": false, "current": { "text": "All", - "value": ["$__all"] + "value": [ + "$__all" + ] }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(slurm_job_info,user_name)", + "definition": "label_values(slurm_job_info{cluster=~\"$cluster\"},user_name)", "description": "", "includeAll": true, "label": "Job Username", @@ -6464,7 +6334,7 @@ "options": [], "query": { "qryType": 5, - "query": "label_values(slurm_job_info,user_name)", + "query": "label_values(slurm_job_info{cluster=~\"$cluster\"},user_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -6503,11 +6373,15 @@ "to": "now" }, "timepicker": { - "refresh_intervals": ["30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"] + "refresh_intervals": [ + "30s", + "1m", + "5m", + "15m" + ] }, "timezone": "browser", "title": "Cluster Health & Overview", "uid": "soperator-cluster-health", - "version": 1, - "weekStart": "" + "version": 2 } diff --git a/helm/soperator-monitoring-dashboards/dashboards/gpu_cluster_stats.json b/helm/soperator-monitoring-dashboards/dashboards/gpu_cluster_stats.json index 5a5dd8128..0844ba6c5 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/gpu_cluster_stats.json +++ b/helm/soperator-monitoring-dashboards/dashboards/gpu_cluster_stats.json @@ -73,7 +73,7 @@ "targets": [ { "editorMode": "code", - "expr": "count(count by (Hostname) (DCGM_FI_DEV_POWER_USAGE{}))", + "expr": "count(count by (Hostname) (DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}))", "instant": true, "legendFormat": "", "range": false, @@ -135,7 +135,7 @@ "targets": [ { "editorMode": "code", - "expr": "count(DCGM_FI_DEV_POWER_USAGE{})", + "expr": "count(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})", "instant": true, "legendFormat": "", "range": false, @@ -193,7 +193,7 @@ "targets": [ { "editorMode": "code", - "expr": "avg_over_time((count(DCGM_FI_DEV_FB_USED{} > 100))[$__range:])", + "expr": "avg_over_time((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"} > 100))[$__range:])", "instant": true, "legendFormat": "", "range": false, @@ -254,7 +254,7 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "avg_over_time((count(DCGM_FI_DEV_FB_USED{} <= 100))[$__range:])", + "expr": "avg_over_time((count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"} <= 100))[$__range:])", "instant": true, "legendFormat": "", "range": false, @@ -321,7 +321,7 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "avg(avg_over_time(DCGM_FI_PROF_SM_ACTIVE{}[$__range])) * 100", + "expr": "avg(avg_over_time(DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}[$__range])) * 100", "instant": true, "legendFormat": "", "range": false, @@ -378,7 +378,7 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "sum(avg_over_time(DCGM_FI_DEV_POWER_USAGE{}[$__range])) / 1000", + "expr": "sum(avg_over_time(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}[$__range])) / 1000", "instant": true, "legendFormat": "", "range": false, @@ -538,21 +538,21 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "avg(DCGM_FI_PROF_SM_ACTIVE{}) * 100", + "expr": "avg(DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}) * 100", "instant": false, "legendFormat": "Average", "range": true, "refId": "A" }, { - "expr": "quantile(0.95, DCGM_FI_PROF_SM_ACTIVE{}) * 100", + "expr": "quantile(0.95, DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}) * 100", "instant": false, "legendFormat": "P95", "range": true, "refId": "B" }, { - "expr": "max(DCGM_FI_PROF_SM_ACTIVE{}) * 100", + "expr": "max(DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}) * 100", "instant": false, "legendFormat": "Max (straggler signal)", "range": true, @@ -670,14 +670,14 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "count(DCGM_FI_DEV_FB_USED{} > 100)", + "expr": "count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"} > 100)", "instant": false, "legendFormat": "In Use", "range": true, "refId": "A" }, { - "expr": "count(DCGM_FI_DEV_FB_USED{} <= 100)", + "expr": "count(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"} <= 100)", "instant": false, "legendFormat": "Idle", "range": true, @@ -765,14 +765,14 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "avg(DCGM_FI_DEV_GPU_UTIL{})", + "expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})", "instant": false, "legendFormat": "Average", "range": true, "refId": "A" }, { - "expr": "min(DCGM_FI_DEV_GPU_UTIL{})", + "expr": "min(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})", "instant": false, "legendFormat": "Min (node stalled?)", "range": true, @@ -860,14 +860,14 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "avg(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{}) * 100", + "expr": "avg(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}) * 100", "instant": false, "legendFormat": "Average", "range": true, "refId": "A" }, { - "expr": "max(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{}) * 100", + "expr": "max(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}) * 100", "instant": false, "legendFormat": "Max", "range": true, @@ -955,7 +955,7 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "avg(DCGM_FI_PROF_SM_OCCUPANCY{}) * 100", + "expr": "avg(DCGM_FI_PROF_SM_OCCUPANCY{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}) * 100", "instant": false, "legendFormat": "Average", "range": true, @@ -1042,7 +1042,7 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "sum(DCGM_FI_DEV_POWER_USAGE{}) / 1000", + "expr": "sum(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}) / 1000", "instant": false, "legendFormat": "Total Power", "range": true, @@ -1160,14 +1160,14 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "sum(DCGM_FI_DEV_FB_USED{})", + "expr": "sum(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})", "instant": false, "legendFormat": "VRAM Used", "range": true, "refId": "A" }, { - "expr": "sum(DCGM_FI_DEV_FB_FREE{})", + "expr": "sum(DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})", "instant": false, "legendFormat": "VRAM Free", "range": true, @@ -1271,7 +1271,7 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "avg by (Hostname) (DCGM_FI_PROF_SM_ACTIVE{}) * 100", + "expr": "avg by (Hostname) (DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}) * 100", "instant": false, "legendFormat": "{{Hostname}}", "range": true, @@ -1344,7 +1344,7 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "avg by (Hostname) (avg_over_time(DCGM_FI_PROF_SM_ACTIVE{}[$__range])) * 100", + "expr": "avg by (Hostname) (avg_over_time(DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}[$__range])) * 100", "instant": true, "legendFormat": "{{Hostname}}", "range": false, @@ -1425,7 +1425,7 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "avg by (Hostname) (DCGM_FI_PROF_SM_ACTIVE{}) * 100", + "expr": "avg by (Hostname) (DCGM_FI_PROF_SM_ACTIVE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}) * 100", "instant": false, "legendFormat": "{{Hostname}}", "range": true, @@ -1569,14 +1569,14 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "max(DCGM_FI_DEV_GPU_TEMP{})", + "expr": "max(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})", "instant": false, "legendFormat": "Max", "range": true, "refId": "A" }, { - "expr": "avg(DCGM_FI_DEV_GPU_TEMP{})", + "expr": "avg(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})", "instant": false, "legendFormat": "Average", "range": true, @@ -1708,7 +1708,7 @@ "targets": [ { "editorMode": "code", - "expr": "max(DCGM_FI_DEV_MEMORY_TEMP{})", + "expr": "max(DCGM_FI_DEV_MEMORY_TEMP{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})", "instant": false, "legendFormat": "Max", "range": true, @@ -1718,7 +1718,7 @@ }, { "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_MEMORY_TEMP{})", + "expr": "avg(DCGM_FI_DEV_MEMORY_TEMP{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})", "instant": false, "legendFormat": "Average", "range": true, @@ -1787,7 +1787,7 @@ "pluginVersion": "11.5.1", "targets": [ { - "expr": "max by (Hostname) (max_over_time(DCGM_FI_DEV_GPU_TEMP{}[$__range]))", + "expr": "max by (Hostname) (max_over_time(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"}[$__range]))", "instant": true, "legendFormat": "{{Hostname}}", "range": false, @@ -1822,6 +1822,74 @@ "skipUrlSync": false, "type": "datasource" }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "allowCustomValue": false, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(slurm_node_info{cluster=~\"$cluster\"},nodeset_name)", + "description": "", + "includeAll": true, + "label": "Nodeset", + "multi": true, + "name": "nodeset", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(slurm_node_info{cluster=~\"$cluster\"},nodeset_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, { "allowCustomValue": false, "current": { @@ -1832,7 +1900,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "query_result(count(count by (Hostname) (DCGM_FI_DEV_POWER_USAGE{})))", + "definition": "query_result(count(count by (Hostname) (DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})))", "hide": 2, "includeAll": false, "label": "Instance count (hidden)", @@ -1846,7 +1914,7 @@ ], "query": { "qryType": 5, - "query": "query_result(count(count by (Hostname) (DCGM_FI_DEV_POWER_USAGE{})))", + "query": "query_result(count(count by (Hostname) (DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\",slurm_nodeset_name=~\"$nodeset\"})))", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "regex": "/\\s(\\d+)\\s*\\d*$/", @@ -1865,6 +1933,6 @@ "timezone": "browser", "title": "GPU Cluster Stats", "uid": "soperator-gpu-cluster-stats", - "version": 1, + "version": 2, "weekStart": "" } diff --git a/helm/soperator-monitoring-dashboards/dashboards/jobs_overview.json b/helm/soperator-monitoring-dashboards/dashboards/jobs_overview.json index 8ed738617..28bee7537 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/jobs_overview.json +++ b/helm/soperator-monitoring-dashboards/dashboards/jobs_overview.json @@ -25,7 +25,7 @@ "uid": "${datasource}" }, "enable": false, - "expr": "group by (node, pod) (\n changes(\n sum without (uid, pod_ip)(\n kube_pod_info{pod=~\"worker-.*\", node!=\"\"}\n )\n ) > 0\n and on (pod) \n sum by (pod)(\n kube_pod_info{pod=~\"worker-.*\", node!=\"\"}\n ) @ start()\n)", + "expr": "group by (node, pod) (\n changes(\n sum without (uid, pod_ip)(\n kube_pod_info{cluster=~\"$cluster\", pod=~\"worker-.*\", node!=\"\"}\n )\n ) > 0\n and on (pod) \n sum by (pod)(\n kube_pod_info{cluster=~\"$cluster\", pod=~\"worker-.*\", node!=\"\"}\n ) @ start()\n)", "hide": true, "iconColor": "red", "name": "Worker Reschedule", @@ -38,14 +38,16 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 20, + "id": 7, "links": [ { "asDropdown": false, "icon": "external link", "includeVars": true, "keepTime": true, - "tags": ["workers-overview"], + "tags": [ + "workers-overview" + ], "targetBlank": true, "title": "Detailed Stats", "tooltip": "", @@ -88,7 +90,7 @@ "content": "* You can select multiple Jobs, list of Workers will be filtered based on that, only one of the jobs is selected by default\n* You can select multiple Workers, All are selected by default\n* Stats in all panels will be based on selected Jobs and Workers\n* check per worker stats below when there are large variance in cluster level stats, or when you want to spot outlier workers\n", "mode": "markdown" }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "title": "Usage Guide", "type": "text" }, @@ -110,8 +112,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "orange", @@ -141,7 +142,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "fields": "", "values": false }, @@ -149,7 +152,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -157,7 +160,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg (\n (1 - rate(node_cpu_seconds_total{mode=\"idle\"}[1m]))\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n) * 100", + "expr": "avg (\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\"}[1m]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n) * 100", "instant": false, "legendFormat": "__auto", "range": true, @@ -189,8 +192,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -217,7 +219,9 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -225,7 +229,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -233,7 +237,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_SM_CLOCK{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}*1000000)", + "expr": "avg(DCGM_FI_DEV_SM_CLOCK{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}*1000000)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -272,8 +276,7 @@ "mode": "absolute", "steps": [ { - "color": "#299c46", - "value": null + "color": "#299c46" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -304,7 +307,9 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -312,7 +317,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -320,7 +325,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_DEV_POWER_USAGE{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "sum(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -357,8 +362,7 @@ "mode": "absolute", "steps": [ { - "color": "#299c46", - "value": null + "color": "#299c46" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -389,7 +393,9 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -397,7 +403,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -405,7 +411,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_MEM_COPY_UTIL{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg(DCGM_FI_DEV_MEM_COPY_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -442,8 +448,7 @@ "mode": "absolute", "steps": [ { - "color": "#299c46", - "value": null + "color": "#299c46" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -474,7 +479,9 @@ "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "fields": "", "values": false }, @@ -482,7 +489,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -490,7 +497,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n (\n node_memory_MemTotal_bytes - \n node_memory_MemFree_bytes - \n node_memory_Buffers_bytes - \n node_memory_Cached_bytes\n ) / \n node_memory_MemTotal_bytes{instance=~\"$ip\"}\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n) * 100", + "expr": "avg(\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n ) / \n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n) * 100", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -524,8 +531,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -552,7 +558,9 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -560,7 +568,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -568,7 +576,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_MEM_CLOCK{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}*1000000)", + "expr": "avg(DCGM_FI_DEV_MEM_CLOCK{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}*1000000)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -609,8 +617,7 @@ "mode": "absolute", "steps": [ { - "color": "#299c46", - "value": null + "color": "#299c46" }, { "color": "orange", @@ -641,7 +648,9 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["mean"], + "calcs": [ + "mean" + ], "fields": "", "values": false }, @@ -649,7 +658,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -657,7 +666,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -694,8 +703,7 @@ "mode": "absolute", "steps": [ { - "color": "#299c46", - "value": null + "color": "#299c46" }, { "color": "rgba(237, 129, 40, 0.89)", @@ -726,7 +734,9 @@ "orientation": "horizontal", "percentChangeColorMode": "standard", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -734,7 +744,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -742,7 +752,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -792,8 +802,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -826,7 +835,7 @@ "properties": [ { "id": "custom.width", - "value": 72 + "value": 94 }, { "id": "custom.align", @@ -842,7 +851,7 @@ "properties": [ { "id": "custom.width", - "value": 67 + "value": 81 } ] }, @@ -854,7 +863,67 @@ "properties": [ { "id": "custom.width", - "value": 87 + "value": 108 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Job State" + }, + "properties": [ + { + "id": "custom.width", + "value": 116 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Job State Reason" + }, + "properties": [ + { + "id": "custom.width", + "value": 169 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Partition" + }, + "properties": [ + { + "id": "custom.width", + "value": 141 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "State" + }, + "properties": [ + { + "id": "custom.width", + "value": 153 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "State Reason" + }, + "properties": [ + { + "id": "custom.width", + "value": 174 } ] } @@ -873,14 +942,16 @@ "countRows": false, "enablePagination": true, "fields": "", - "reducer": ["sum"], + "reducer": [ + "sum" + ], "show": false }, "frameIndex": 0, "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -889,7 +960,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "with (\n filters = {job_id=~\"$slurm_job\"}\n)\nslurm_job_info{filters}\n* on (job_id) group_left()\ncount by (job_id) (slurm_node_job{filters}) ", + "expr": "with (\n filters = {job_id=~\"$slurm_job\"}\n)\nslurm_job_info{cluster=~\"$cluster\", filters}\n* on (job_id) group_left()\ncount by (job_id) (slurm_node_job{cluster=~\"$cluster\", filters}) ", "format": "table", "instant": true, "range": false, @@ -926,16 +997,21 @@ "container_id": true, "end_time": true, "endpoint": true, + "finished_time": true, "finished_time (first)": true, + "iam_project_id": true, + "iam_tenant_id": true, "instance": true, "job": true, - "job_state": true, - "job_state_reason": true, + "job_state": false, + "job_state_reason": false, + "mk8s_cluster_id": true, "namespace": true, + "nodeset_name (uniqueValues)": false, "pod": true, "prometheus": true, "slurm_job_info (last)": true, - "slurm_partition": true, + "slurm_partition": false, "standard_error": true, "standard_output": true, "start_time": true, @@ -949,37 +1025,42 @@ "Value": 3, "array_job_id": 5, "cluster": 6, - "container": 7, - "container_id": 8, - "end_time": 9, - "endpoint": 10, - "instance": 11, - "job": 12, + "container_id": 7, + "end_time": 8, + "endpoint": 9, + "finished_time": 18, + "iam_project_id": 19, + "iam_tenant_id": 20, + "job": 10, "job_id": 0, - "job_name": 13, - "job_state": 14, - "job_state_reason": 15, - "namespace": 16, - "pod": 17, - "prometheus": 18, - "slurm_partition": 19, - "standard_error": 20, - "standard_output": 21, - "start_time": 22, - "submit_time": 23, + "job_name": 22, + "job_state": 11, + "job_state_reason": 12, + "mk8s_cluster_id": 21, + "namespace": 13, + "prometheus": 14, + "slurm_partition": 15, + "start_time": 16, + "submit_time": 17, "user_id": 1, "user_name": 2 }, "renameByName": { "Value": "Workers", + "Value #A": "Workers", "container": "", + "finished_time": "Finish Time", "finished_time_ms": "Finished Time", "job": "", "job_id": "Job ID", "job_name": "Job Name", "job_name (first)": "Job Name", + "job_state": "State", "job_state (first)": "State", + "job_state_reason": "State Reason", "job_state_reason (first)": "State Reason", + "nodeset_name (uniqueValues)": "Nodesets", + "slurm_partition": "Partition", "slurm_partition (first)": "Partition", "start_time_ms": "Start Time", "submit_time": "", @@ -1005,6 +1086,7 @@ "mode": "palette-classic-by-name" }, "custom": { + "axisPlacement": "auto", "fillOpacity": 75, "hideFrom": { "legend": false, @@ -1131,8 +1213,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1167,7 +1248,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1175,7 +1256,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "with (\n filters = { job_id=~\"$slurm_job\"},\n)\nsum by (job_id,)(\n count by (job_id, job_name) (slurm_job_info{job_state=\"PENDING\", filters}) * 1 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"CONFIGURING\",filters}) * 2 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"RUNNING\", filters}) * 3 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"COMPLETING\", filters}) * 4 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"COMPLETED\", filters}) * 5 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"CANCELLED\", filters}) * 6 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"FAILED\", filters}) * 7 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"TIMEOUT\", filters}) * 8 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"PREEMPTED\", filters}) * 9 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"NODE_FAIL\", filters}) * 10 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"BOOT_FAIL\", filters}) * 11 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"DEADLINE\", filters}) * 12 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"OUT_OF_MEMORY\", filters}) * 13 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"REQUEUED\", filters}) * 14 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"RESIZING\", filters}) * 15 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"SUSPENDED\", filters}) * 16 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"STOPPED\", filters}) * 17 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"POWER_UP_NODE\", filters}) * 18 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"SIGNALING\", filters}) * 19 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"STAGE_OUT\", filters}) * 20 or\n count by (job_id, job_name) (slurm_job_info{job_state=\"UPDATE_DB\", filters}) * 21\n)", + "expr": "with (\n filters = { job_id=~\"$slurm_job\"},\n)\nsum by (job_id,)(\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"PENDING\", filters}) * 1 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"CONFIGURING\",filters}) * 2 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"RUNNING\", filters}) * 3 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"COMPLETING\", filters}) * 4 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"COMPLETED\", filters}) * 5 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"CANCELLED\", filters}) * 6 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"FAILED\", filters}) * 7 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"TIMEOUT\", filters}) * 8 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"PREEMPTED\", filters}) * 9 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"NODE_FAIL\", filters}) * 10 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"BOOT_FAIL\", filters}) * 11 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"DEADLINE\", filters}) * 12 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"OUT_OF_MEMORY\", filters}) * 13 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"REQUEUED\", filters}) * 14 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"RESIZING\", filters}) * 15 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"SUSPENDED\", filters}) * 16 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"STOPPED\", filters}) * 17 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"POWER_UP_NODE\", filters}) * 18 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"SIGNALING\", filters}) * 19 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"STAGE_OUT\", filters}) * 20 or\n count by (job_id, job_name) (slurm_job_info{cluster=~\"$cluster\", job_state=\"UPDATE_DB\", filters}) * 21\n)", "format": "time_series", "hide": false, "legendFormat": "{{job_id}}", @@ -1253,8 +1334,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1286,7 +1366,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -1320,7 +1403,9 @@ "interval": "2m", "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -1331,7 +1416,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1339,7 +1424,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "max(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1354,7 +1439,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1369,7 +1454,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "min(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1431,8 +1516,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1449,7 +1533,9 @@ "id": 405, "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -1460,7 +1546,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1468,7 +1554,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"} * 1979)", + "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"} * 1979)", "format": "time_series", "hide": false, "interval": "", @@ -1483,7 +1569,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"} * 989.7)", + "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"} * 989.7)", "format": "time_series", "hide": false, "interval": "", @@ -1550,8 +1636,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1583,7 +1668,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -1632,7 +1720,9 @@ "interval": "2m", "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -1643,7 +1733,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1651,7 +1741,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max (DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "max (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1666,7 +1756,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg (DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1681,7 +1771,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min (DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "min (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -1745,8 +1835,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1778,7 +1867,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -1823,7 +1915,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -1831,7 +1923,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(\n (1 - rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "max(\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "instant": false, "legendFormat": "max", "range": true, @@ -1843,7 +1935,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n (1 - rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "avg(\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "hide": false, "instant": false, "legendFormat": "avg", @@ -1856,7 +1948,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(\n (1 - rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "min(\n (1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "hide": false, "instant": false, "legendFormat": "min", @@ -1919,8 +2011,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1952,7 +2043,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -1994,7 +2088,9 @@ "interval": "2m", "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2005,7 +2101,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -2013,7 +2109,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "max(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", "format": "time_series", "hide": false, "interval": "", @@ -2028,7 +2124,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "avg(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", "format": "time_series", "hide": false, "interval": "", @@ -2043,7 +2139,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "min(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", "format": "time_series", "hide": false, "interval": "", @@ -2105,8 +2201,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2163,7 +2258,9 @@ "interval": "1m", "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2174,7 +2271,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -2182,7 +2279,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(node_infiniband_port_data_transmitted_bytes_total{instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "sum(\n rate(node_infiniband_port_data_transmitted_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "interval": "", @@ -2197,7 +2294,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(node_infiniband_port_data_received_bytes_total{instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "sum(\n rate(node_infiniband_port_data_received_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -2261,8 +2358,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2294,7 +2390,10 @@ { "id": "custom.lineStyle", "value": { - "dash": [10, 10], + "dash": [ + 10, + 10 + ], "fill": "dash" } }, @@ -2347,7 +2446,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -2355,7 +2454,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(\n (\n (\n node_memory_MemTotal_bytes - \n node_memory_MemFree_bytes - \n node_memory_Buffers_bytes - \n node_memory_Cached_bytes\n )\n /\n node_memory_MemTotal_bytes{instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "max(\n (\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -2370,7 +2469,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n (\n (\n node_memory_MemTotal_bytes - \n node_memory_MemFree_bytes - \n node_memory_Buffers_bytes - \n node_memory_Cached_bytes\n )\n /\n node_memory_MemTotal_bytes{instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "avg(\n (\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -2385,7 +2484,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(\n (\n (\n node_memory_MemTotal_bytes - \n node_memory_MemFree_bytes - \n node_memory_Buffers_bytes - \n node_memory_Cached_bytes\n )\n /\n node_memory_MemTotal_bytes{instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "min(\n (\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n )\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(hpc_job) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -2466,8 +2565,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2485,7 +2583,9 @@ "interval": "2m", "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2496,7 +2596,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -2504,7 +2604,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2569,8 +2669,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2588,7 +2687,9 @@ "interval": "2m", "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2599,7 +2700,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -2607,7 +2708,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", "format": "time_series", "hide": false, "interval": "", @@ -2674,8 +2775,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2693,7 +2793,9 @@ "interval": "2m", "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2704,7 +2806,7 @@ "sort": "none" } }, - "pluginVersion": "11.4.0", + "pluginVersion": "11.6.14+security-04", "targets": [ { "datasource": { @@ -2712,7 +2814,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_MEM_COPY_UTIL{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_MEM_COPY_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2774,8 +2876,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2793,7 +2894,9 @@ "interval": "2m", "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2812,7 +2915,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (exported_pod) (DCGM_FI_DEV_POWER_USAGE{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "sum by (exported_pod) (DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2877,8 +2980,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -2896,7 +2998,9 @@ "interval": "2m", "options": { "legend": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "displayMode": "list", "placement": "bottom", "showLegend": false @@ -2915,7 +3019,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"})", "format": "time_series", "hide": false, "interval": "", @@ -2978,8 +3082,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -3031,7 +3134,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (err_msg, exported_pod) (rate(DCGM_FI_DEV_XID_ERRORS{exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", + "expr": "sum by (err_msg, exported_pod) (rate(DCGM_FI_DEV_XID_ERRORS{cluster=~\"$cluster\", exported_pod=~\"$worker\",hpc_job=~\"$slurm_job\"}))", "instant": false, "legendFormat": "{{exported_pod}}: {{err_msg}}", "range": true, @@ -3044,28 +3147,68 @@ ], "preload": false, "refresh": "", - "schemaVersion": 40, - "tags": ["nebius", "soperator", "overview", "jobs"], + "schemaVersion": 41, + "tags": [ + "nebius", + "soperator", + "overview", + "jobs" + ], "templating": { "list": [ { "current": { - "selected": true, "text": "VictoriaMetrics", "value": "VictoriaMetrics" }, "hide": 2, "includeAll": false, "label": "Datasource", - "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, { "current": { "text": "1", @@ -3087,14 +3230,24 @@ }, { "current": { - "text": ["102"], - "value": ["102"] + "text": [ + "86", + "85", + "84", + "83" + ], + "value": [ + "86", + "85", + "84", + "83" + ] }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "definition": "query_result(count by (job_id) (slurm_job_info{user_name!=\"soperatorchecks\"}))", + "definition": "query_result(count by (job_id) (slurm_job_info{cluster=~\"$cluster\", user_name!=\"soperatorchecks\"}))", "description": "", "includeAll": false, "label": "Jobs", @@ -3103,7 +3256,7 @@ "options": [], "query": { "qryType": 3, - "query": "query_result(count by (job_id) (slurm_job_info{user_name!=\"soperatorchecks\"}))", + "query": "query_result(count by (job_id) (slurm_job_info{cluster=~\"$cluster\", user_name!=\"soperatorchecks\"}))", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, @@ -3112,6 +3265,7 @@ "type": "query" }, { + "allValue": ".*", "current": { "text": "All", "value": "$__all" @@ -3120,7 +3274,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"},exported_pod)", + "definition": "label_values(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"},exported_pod)", "description": "Filtered by selected Jobs", "includeAll": true, "label": "Workers", @@ -3129,7 +3283,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(DCGM_FI_DEV_GPU_TEMP{hpc_job=~\"$slurm_job\"},exported_pod)", + "query": "label_values(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job=~\"$slurm_job\"},exported_pod)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, @@ -3138,6 +3292,7 @@ "type": "query" }, { + "allValue": ".*", "current": { "text": "All", "value": "$__all" @@ -3146,7 +3301,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(kube_pod_info{pod=~\"$worker\"},node)", + "definition": "label_values(kube_pod_info{cluster=~\"$cluster\", pod=~\"$worker\"},node)", "hide": 2, "includeAll": true, "label": "instance", @@ -3155,7 +3310,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(kube_pod_info{pod=~\"$worker\"},node)", + "query": "label_values(kube_pod_info{cluster=~\"$cluster\", pod=~\"$worker\"},node)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, @@ -3164,6 +3319,7 @@ "type": "query" }, { + "allValue": ".*", "current": { "text": "All", "value": "$__all" @@ -3172,7 +3328,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(node_uname_info{container=\"node-exporter\", nodename=~\"$hostname\"},instance)", + "definition": "label_values(node_uname_info{cluster=~\"$cluster\", container=\"node-exporter\", nodename=~\"$hostname\"},instance)", "hide": 2, "includeAll": true, "label": "ip", @@ -3181,7 +3337,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(node_uname_info{container=\"node-exporter\", nodename=~\"$hostname\"},instance)", + "query": "label_values(node_uname_info{cluster=~\"$cluster\", container=\"node-exporter\", nodename=~\"$hostname\"},instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -3210,6 +3366,5 @@ "timezone": "browser", "title": "Jobs Overview", "uid": "soperator_jobs_overview", - "version": 1, - "weekStart": "" + "version": 2 } diff --git a/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_job_performance.json b/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_job_performance.json index 0c60aa440..7d31fbc1d 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_job_performance.json +++ b/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_job_performance.json @@ -126,7 +126,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_p2p_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", p2p_operation=\"Recv\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_p2p_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", p2p_operation=\"Recv\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -233,7 +233,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_p2p_exec_time_microseconds{slurm_job_id=~\"$jobid\", p2p_operation=\"Recv\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_p2p_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", p2p_operation=\"Recv\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -340,7 +340,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_p2p_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", p2p_operation=\"Recv\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_p2p_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", p2p_operation=\"Recv\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -447,7 +447,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_p2p_exec_time_microseconds{slurm_job_id=~\"$jobid\", p2p_operation=\"Recv\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_p2p_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", p2p_operation=\"Recv\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -568,7 +568,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_p2p_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", p2p_operation=\"Send\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_p2p_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", p2p_operation=\"Send\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -675,7 +675,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_p2p_exec_time_microseconds{slurm_job_id=~\"$jobid\", p2p_operation=\"Send\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_p2p_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", p2p_operation=\"Send\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -782,7 +782,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_p2p_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", p2p_operation=\"Send\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_p2p_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", p2p_operation=\"Send\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -889,7 +889,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_p2p_exec_time_microseconds{slurm_job_id=~\"$jobid\", p2p_operation=\"Send\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_p2p_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", p2p_operation=\"Send\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -1013,7 +1013,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", collective=\"ReduceScatter\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"ReduceScatter\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -1120,7 +1120,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{slurm_job_id=~\"$jobid\", collective=\"ReduceScatter\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"ReduceScatter\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -1227,7 +1227,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", collective=\"ReduceScatter\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"ReduceScatter\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -1334,7 +1334,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{slurm_job_id=~\"$jobid\", collective=\"ReduceScatter\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"ReduceScatter\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -1454,7 +1454,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", collective=\"AllReduce\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"AllReduce\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -1561,7 +1561,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{slurm_job_id=~\"$jobid\", collective=\"AllReduce\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"AllReduce\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -1668,7 +1668,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", collective=\"AllReduce\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"AllReduce\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -1775,7 +1775,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{slurm_job_id=~\"$jobid\", collective=\"AllReduce\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"AllReduce\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -1895,7 +1895,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", collective=\"AllGather\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"AllGather\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -2002,7 +2002,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{slurm_job_id=~\"$jobid\", collective=\"AllGather\", n_nodes=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"AllGather\", n_nodes=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}", @@ -2109,7 +2109,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{slurm_job_id=~\"$jobid\", collective=\"AllGather\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"AllGather\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -2216,7 +2216,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{slurm_job_id=~\"$jobid\", collective=\"AllGather\", n_nodes!=\"1\", hostname=~\"$hostname\"})", + "expr": "avg by(message_size, nranks, n_nodes, hostname, slurm_job_id) (nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", slurm_job_id=~\"$jobid\", collective=\"AllGather\", n_nodes!=\"1\", hostname=~\"$hostname\"})", "fullMetaSearch": false, "includeNullMetadata": true, "legendFormat": "{{hostname}} - {{slurm_job_id}} | message_size: {{message_size}}, nranks: {{nranks}}, n_nodes: {{n_nodes}}", @@ -2255,6 +2255,46 @@ "skipUrlSync": false, "type": "datasource" }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, { "current": { "text": [ @@ -2268,15 +2308,16 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({__name__=~\"nccl_.*\", slurm_job_id!=\"\"}, slurm_job_id)", + "definition": "label_values({cluster=~\"$cluster\", __name__=~\"nccl_.*\", slurm_job_id!=\"\"}, slurm_job_id)", "includeAll": true, + "allValue": ".*", "label": "Slurm Job ID", "multi": true, "name": "jobid", "options": [], "query": { "qryType": 5, - "query": "label_values({__name__=~\"nccl_.*\", slurm_job_id!=\"\"}, slurm_job_id)", + "query": "label_values({cluster=~\"$cluster\", __name__=~\"nccl_.*\", slurm_job_id!=\"\"}, slurm_job_id)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -2296,15 +2337,16 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({__name__=~\"nccl_.*\", hostname!=\"\"}, hostname)", + "definition": "label_values({cluster=~\"$cluster\", __name__=~\"nccl_.*\", hostname!=\"\"}, hostname)", "includeAll": true, + "allValue": ".*", "label": "Worker", "multi": true, "name": "hostname", "options": [], "query": { "qryType": 5, - "query": "label_values({__name__=~\"nccl_.*\", hostname!=\"\"}, hostname)", + "query": "label_values({cluster=~\"$cluster\", __name__=~\"nccl_.*\", hostname!=\"\"}, hostname)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, diff --git a/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_metrics.json b/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_metrics.json index 2a5001ad7..795573eed 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_metrics.json +++ b/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_metrics.json @@ -132,7 +132,7 @@ "disableTextWrap": false, "editorMode": "builder", "exemplar": false, - "expr": "nccl_algorithm_bandwidth_gbs{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", + "expr": "nccl_algorithm_bandwidth_gbs{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, @@ -259,7 +259,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "nccl_bus_bandwidth_gbs{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"${collective}\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", + "expr": "nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"${collective}\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, @@ -346,7 +346,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "nccl_bus_bandwidth_gbs{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", + "expr": "nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, @@ -443,7 +443,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "nccl_bus_bandwidth_gbs{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", + "expr": "nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, @@ -568,7 +568,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "nccl_collective_exec_time_microseconds{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"} < 50000", + "expr": "nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"} < 50000", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, @@ -653,7 +653,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "nccl_collective_exec_time_microseconds{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"} < 50000", + "expr": "nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"} < 50000", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, @@ -739,7 +739,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "nccl_collective_exec_time_microseconds{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"} < 50000", + "expr": "nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"} < 50000", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, @@ -845,7 +845,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "nccl_collective_exec_time_microseconds{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"} > 50000", + "expr": "nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"} > 50000", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, @@ -968,7 +968,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "nccl_message_size_bytes{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", + "expr": "nccl_message_size_bytes{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, @@ -1090,7 +1090,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "nccl_collective_sequence_number{rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", + "expr": "nccl_collective_sequence_number{cluster=~\"$cluster\", rank=~\"${rank}\", nranks=~\"${n_ranks}\", collective=~\"$collective\", slurm_job_id=~\"$jobid\", hostname=~\"$hostname\"}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, @@ -1132,6 +1132,46 @@ "skipUrlSync": false, "type": "datasource" }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, { "allValue": ".*", "current": { @@ -1142,14 +1182,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({__name__=~\"nccl.*\", rank!=\"\"}, rank)", + "definition": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", rank!=\"\"}, rank)", "includeAll": true, "label": "Rank", "name": "rank", "options": [], "query": { "qryType": 5, - "query": "label_values({__name__=~\"nccl.*\", rank!=\"\"}, rank)", + "query": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", rank!=\"\"}, rank)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -1167,14 +1207,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({__name__=~\"nccl.*\", nranks!=\"\"}, nranks)", + "definition": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", nranks!=\"\"}, nranks)", "includeAll": true, "label": "N Ranks", "name": "n_ranks", "options": [], "query": { "qryType": 5, - "query": "label_values({__name__=~\"nccl.*\", nranks!=\"\"}, nranks)", + "query": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", nranks!=\"\"}, nranks)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -1192,14 +1232,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({__name__=~\"nccl.*\", collective!=\"\"}, collective)", + "definition": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", collective!=\"\"}, collective)", "includeAll": true, "label": "Collective", "name": "collective", "options": [], "query": { "qryType": 5, - "query": "label_values({__name__=~\"nccl.*\", collective!=\"\"}, collective)", + "query": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", collective!=\"\"}, collective)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -1220,15 +1260,16 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({__name__=~\"nccl.*\", slurm_job_id!=\"\"}, slurm_job_id)", + "definition": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", slurm_job_id!=\"\"}, slurm_job_id)", "includeAll": true, + "allValue": ".*", "label": "Slurm Job ID", "multi": true, "name": "jobid", "options": [], "query": { "qryType": 5, - "query": "label_values({__name__=~\"nccl.*\", slurm_job_id!=\"\"}, slurm_job_id)", + "query": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", slurm_job_id!=\"\"}, slurm_job_id)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -1248,15 +1289,16 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({__name__=~\"nccl.*\", hostname!=\"\"}, hostname)", + "definition": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", hostname!=\"\"}, hostname)", "includeAll": true, + "allValue": ".*", "label": "Worker", "multi": true, "name": "hostname", "options": [], "query": { "qryType": 5, - "query": "label_values({__name__=~\"nccl.*\", hostname!=\"\"}, hostname)", + "query": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", hostname!=\"\"}, hostname)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, diff --git a/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_raw_metrics.json b/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_raw_metrics.json index 74dd95ce1..0ef69b182 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_raw_metrics.json +++ b/helm/soperator-monitoring-dashboards/dashboards/nccl_inspector_raw_metrics.json @@ -115,7 +115,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "nccl_algorithm_bandwidth_gbs{hostname=~\"$hostname\", slurm_job_id=~\"$jobid\"}", + "expr": "nccl_algorithm_bandwidth_gbs{cluster=~\"$cluster\", hostname=~\"$hostname\", slurm_job_id=~\"$jobid\"}", "interval": "100ms", "legendFormat": "{{slurm_job_id}}.{{slurm_step_id}} | {{hostname}} [{{hostname_pid}}] / {{n_nodes}} | {{collective}} | R {{rank}}/{{nranks}} | {{message_size_bytes}} B | {{comm_id}}", "range": true, @@ -221,7 +221,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "nccl_bus_bandwidth_gbs{hostname=~\"$hostname\", slurm_job_id=~\"$jobid\"}", + "expr": "nccl_bus_bandwidth_gbs{cluster=~\"$cluster\", hostname=~\"$hostname\", slurm_job_id=~\"$jobid\"}", "interval": "100ms", "legendFormat": "{{slurm_job_id}}.{{slurm_step_id}} | {{hostname}} [{{hostname_pid}}] / {{n_nodes}} | {{collective}} | R {{rank}}/{{nranks}} | {{message_size_bytes}} B | {{comm_id}}", "range": true, @@ -327,7 +327,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "nccl_collective_exec_time_microseconds{hostname=~\"$hostname\", slurm_job_id=~\"$jobid\"}", + "expr": "nccl_collective_exec_time_microseconds{cluster=~\"$cluster\", hostname=~\"$hostname\", slurm_job_id=~\"$jobid\"}", "interval": "100ms", "legendFormat": "{{slurm_job_id}}.{{slurm_step_id}} | {{hostname}} [{{hostname_pid}}] / {{n_nodes}} | {{collective}} | R {{rank}}/{{nranks}} | {{message_size_bytes}} B | {{comm_id}}", "range": true, @@ -433,7 +433,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "nccl_message_size_bytes{hostname=~\"$hostname\", slurm_job_id=~\"$jobid\"}", + "expr": "nccl_message_size_bytes{cluster=~\"$cluster\", hostname=~\"$hostname\", slurm_job_id=~\"$jobid\"}", "interval": "100ms", "legendFormat": "{{slurm_job_id}}.{{slurm_step_id}} | {{hostname}} [{{hostname_pid}}] / {{n_nodes}} | {{collective}} | R {{rank}}/{{nranks}} | {{message_size_bytes}} B | {{comm_id}}", "range": true, @@ -472,6 +472,46 @@ "skipUrlSync": false, "type": "datasource" }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, { "allowCustomValue": false, "current": { @@ -486,15 +526,16 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({__name__=~\"nccl.*\", slurm_job_id!=\"\"}, slurm_job_id)", + "definition": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", slurm_job_id!=\"\"}, slurm_job_id)", "includeAll": true, + "allValue": ".*", "label": "Slurm Job ID", "multi": true, "name": "jobid", "options": [], "query": { "qryType": 5, - "query": "label_values({__name__=~\"nccl.*\", slurm_job_id!=\"\"}, slurm_job_id)", + "query": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", slurm_job_id!=\"\"}, slurm_job_id)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -515,15 +556,16 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values({__name__=~\"nccl.*\", hostname!=\"\"}, hostname)", + "definition": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", hostname!=\"\"}, hostname)", "includeAll": true, + "allValue": ".*", "label": "Worker", "multi": true, "name": "hostname", "options": [], "query": { "qryType": 5, - "query": "label_values({__name__=~\"nccl.*\", hostname!=\"\"}, hostname)", + "query": "label_values({cluster=~\"$cluster\", __name__=~\"nccl.*\", hostname!=\"\"}, hostname)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, diff --git a/helm/soperator-monitoring-dashboards/dashboards/nfs_server_client.json b/helm/soperator-monitoring-dashboards/dashboards/nfs_server_client.json index 2362269b9..7e0e4fdfe 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/nfs_server_client.json +++ b/helm/soperator-monitoring-dashboards/dashboards/nfs_server_client.json @@ -85,7 +85,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_nfsd_server_threads{ namespace=\"nfs-system\"}", + "expr": "node_nfsd_server_threads{cluster=~\"$cluster\", namespace=\"nfs-system\"}", "legendFormat": "__auto", "range": true, "refId": "A", @@ -162,7 +162,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "kubelet_volume_stats_used_bytes{\n persistentvolumeclaim=\"nfs-server-storage\"\n}\n/\nkubelet_volume_stats_capacity_bytes{\n persistentvolumeclaim=\"nfs-server-storage\"\n}", + "expr": "kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", \n persistentvolumeclaim=\"nfs-server-storage\"\n}\n/\nkubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", \n persistentvolumeclaim=\"nfs-server-storage\"\n}", "range": true, "refId": "A", "useCustomValues": false, @@ -239,7 +239,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "kubelet_volume_stats_capacity_bytes{\n persistentvolumeclaim=\"nfs-server-storage\"\n}", + "expr": "kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", \n persistentvolumeclaim=\"nfs-server-storage\"\n}", "range": true, "refId": "A", "useCustomValues": false, @@ -316,7 +316,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (cluster) (rate(node_nfs_rpc_retransmissions_total{ }[$__rate_interval]))", + "expr": "sum by (cluster) (rate(node_nfs_rpc_retransmissions_total{cluster=~\"$cluster\" }[$__rate_interval]))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -446,7 +446,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_nfs_rpcs_total{ }[$__rate_interval])\n* on(cluster, pod) group_left(node)\nkube_pod_info{ node=~\"$compute_instance\"}", + "expr": "rate(node_nfs_rpcs_total{cluster=~\"$cluster\" }[$__rate_interval])\n* on(cluster, pod) group_left(node)\nkube_pod_info{cluster=~\"$cluster\", node=~\"$compute_instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -563,7 +563,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_nfs_rpc_retransmissions_total{ }[$__rate_interval])\n* on(cluster, pod) group_left(node)\nkube_pod_info{ node=~\"$compute_instance\"}", + "expr": "rate(node_nfs_rpc_retransmissions_total{cluster=~\"$cluster\" }[$__rate_interval])\n* on(cluster, pod) group_left(node)\nkube_pod_info{cluster=~\"$cluster\", node=~\"$compute_instance\"}", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -682,7 +682,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(node_nfs_requests_total{ proto=\"4\"}[$__rate_interval])\n * on(cluster, pod) group_left(node)\n kube_pod_info{ node=~\"$compute_instance\"}\n) by (method)", + "expr": "sum(\n rate(node_nfs_requests_total{cluster=~\"$cluster\", proto=\"4\"}[$__rate_interval])\n * on(cluster, pod) group_left(node)\n kube_pod_info{cluster=~\"$cluster\", node=~\"$compute_instance\"}\n) by (method)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -871,7 +871,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_nfsd_disk_bytes_read_total{ namespace=\"nfs-system\"}[$__rate_interval])", + "expr": "rate(node_nfsd_disk_bytes_read_total{cluster=~\"$cluster\", namespace=\"nfs-system\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "NFSd bytes read", @@ -887,7 +887,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_nfsd_disk_bytes_written_total{ namespace=\"nfs-system\"}[$__rate_interval])", + "expr": "rate(node_nfsd_disk_bytes_written_total{cluster=~\"$cluster\", namespace=\"nfs-system\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "NFSd bytes written", @@ -1051,7 +1051,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_nfsd_packets_total{proto=\"tcp\", namespace=\"nfs-system\"}[$__rate_interval])", + "expr": "rate(node_nfsd_packets_total{cluster=~\"$cluster\", proto=\"tcp\", namespace=\"nfs-system\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "TCP - NFSd network packets (sent+received)", @@ -1067,7 +1067,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_nfsd_packets_total{proto=\"udp\", namespace=\"nfs-system\"}[$__rate_interval])", + "expr": "rate(node_nfsd_packets_total{cluster=~\"$cluster\", proto=\"udp\", namespace=\"nfs-system\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "UDP - NFSd network packets (sent+received)", @@ -1169,7 +1169,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(node_nfsd_read_ahead_cache_not_found_total{ namespace=\"nfs-system\" }[$__rate_interval])", + "expr": "rate(node_nfsd_read_ahead_cache_not_found_total{cluster=~\"$cluster\", namespace=\"nfs-system\" }[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1183,7 +1183,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(node_nfsd_reply_cache_hits_total{ namespace=\"nfs-system\" }[$__rate_interval])", + "expr": "rate(node_nfsd_reply_cache_hits_total{cluster=~\"$cluster\", namespace=\"nfs-system\" }[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1197,7 +1197,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(node_nfsd_reply_cache_misses_total{ namespace=\"nfs-system\" }[$__rate_interval])", + "expr": "rate(node_nfsd_reply_cache_misses_total{cluster=~\"$cluster\", namespace=\"nfs-system\" }[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1211,7 +1211,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(node_nfsd_reply_cache_nocache_total{ namespace=\"nfs-system\" }[$__rate_interval])", + "expr": "rate(node_nfsd_reply_cache_nocache_total{cluster=~\"$cluster\", namespace=\"nfs-system\" }[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1324,7 +1324,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(node_nfsd_rpc_errors_total{ namespace=\"nfs-system\" }[$__rate_interval])", + "expr": "rate(node_nfsd_rpc_errors_total{cluster=~\"$cluster\", namespace=\"nfs-system\" }[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1338,7 +1338,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(node_nfsd_server_rpcs_total{ namespace=\"nfs-system\" }[$__rate_interval])", + "expr": "rate(node_nfsd_server_rpcs_total{cluster=~\"$cluster\", namespace=\"nfs-system\" }[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -1439,7 +1439,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_nfsd_requests_total{proto=\"4\", namespace=\"nfs-system\" }[$__rate_interval])", + "expr": "rate(node_nfsd_requests_total{cluster=~\"$cluster\", proto=\"4\", namespace=\"nfs-system\" }[$__rate_interval])", "intervalFactor": 1, "legendFormat": "{{ method }}", "range": true, @@ -1477,6 +1477,46 @@ "skipUrlSync": false, "type": "datasource" }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, { "allValue": ".*", "allowCustomValue": true, @@ -1488,7 +1528,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(kube_pod_info{}, node)", + "definition": "label_values(kube_pod_info{cluster=~\"$cluster\"}, node)", "description": "NFS client compute instance", "includeAll": true, "label": "Compute instance", @@ -1497,7 +1537,7 @@ "options": [], "query": { "qryType": 5, - "query": "label_values(kube_pod_info{}, node)", + "query": "label_values(kube_pod_info{cluster=~\"$cluster\"}, node)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, diff --git a/helm/soperator-monitoring-dashboards/dashboards/nodesets_overview.json b/helm/soperator-monitoring-dashboards/dashboards/nodesets_overview.json new file mode 100644 index 000000000..e11cfe62d --- /dev/null +++ b/helm/soperator-monitoring-dashboards/dashboards/nodesets_overview.json @@ -0,0 +1,1328 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "For GB platforms", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": 26, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "soperator", + "gpu" + ], + "targetBlank": true, + "title": "GPU Stats", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 108, + "panels": [], + "title": "Quick Stats", + "type": "row" + }, + { + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 111, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "On GB platforms we create one Slurm Nodeset and one NVLink Instance Group per physical rack (18 nodes).
On other platforms Nodesets can have anywhere between 1 and 100 nodes.", + "mode": "html" + }, + "pluginVersion": "11.6.14+security-04", + "title": "", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "For GB platform only", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "super-light-purple" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 102, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.14+security-04", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "count(\n count(slurm_node_nvlink_instance_group{cluster=~\"$cluster\"}) by (cluster, nvlink_instance_group)\n) by (cluster)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useCustomValues": true, + "useDashValues": false + } + ], + "title": "NVLink Instance Groups", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "GPU Nodes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "super-light-purple" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.14+security-04", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "count(\n slurm_node_info{cluster=~\"$cluster\"}\n) by()", + "instant": true, + "range": false, + "refId": "A", + "useCustomValues": true, + "useDashValues": false + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "super-light-purple" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.14+security-04", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"}\n) by()", + "instant": true, + "range": false, + "refId": "A", + "useCustomValues": true, + "useDashValues": false + } + ], + "title": "GPUs", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "No GPU", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "super-light-blue" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 3, + "maxPerRow": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.6.14+security-04", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "group(\n kube_node_labels{cluster=~\"$cluster\", label_nebius_com_gpu_name!=\"\", label_topology_nebius_com_nvl_instance_group_id!=\"\"}\n) by (label_nebius_com_gpu_name)", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A", + "useCustomValues": false, + "useDashValues": false + } + ], + "title": "GPU Platform", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 109, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Mapping Nodeset to NVLIG to Instance ID", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Nodeset" + }, + "properties": [ + { + "id": "custom.width", + "value": 516 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "NVLink Instance Group ID" + }, + "properties": [ + { + "id": "custom.width", + "value": 672 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Instance Count" + }, + "properties": [ + { + "id": "custom.width", + "value": 164 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 16, + "x": 0, + "y": 5 + }, + "id": 107, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.6.14+security-04", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "group (slurm_node_info{cluster=~\"$cluster\"}) by (node_name, nodeset_name, state_base, state_is_maintenance, state_is_not_responding, reason)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "group (slurm_node_nvlink_instance_group{cluster=~\"$cluster\"}) by (node_name, nodeset_name, nvlink_instance_group, instance_id)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Nodes", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "node_name", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Value": true, + "Value #A": true, + "Value #B": true, + "__name__": true, + "__name__ 1": true, + "__name__ 2": true, + "address": true, + "cluster": true, + "cluster 1": true, + "cluster 2": true, + "container_id": true, + "container_id 1": true, + "container_id 2": true, + "endpoint": true, + "endpoint 1": true, + "endpoint 2": true, + "iam_project_id": true, + "iam_project_id 1": true, + "iam_project_id 2": true, + "iam_tenant_id": true, + "iam_tenant_id 1": true, + "iam_tenant_id 2": true, + "instance_id 1": true, + "instance_id 2": true, + "job": true, + "job 1": true, + "job 2": true, + "mk8s_cluster_id": true, + "mk8s_cluster_id 1": true, + "mk8s_cluster_id 2": true, + "namespace": true, + "namespace 1": true, + "namespace 2": true, + "node_name 1": true, + "nodeset_name 1": false, + "nodeset_name 2": true, + "prometheus": true, + "prometheus 1": true, + "prometheus 2": true, + "state_base": false, + "state_is_cloud": true, + "state_is_drain": true, + "state_is_maintenance": true, + "state_is_reserved": true + }, + "includeByName": {}, + "indexByName": { + "Time 1": 4, + "Time 2": 8, + "Value #A": 10, + "Value #B": 7, + "instance_id": 1, + "node_name": 0, + "nodeset_name 1": 2, + "nodeset_name 2": 9, + "nvlink_instance_group": 3, + "state_base": 5, + "state_is_maintenance": 6 + }, + "renameByName": { + "Time 1": "", + "instance_id": "Compute Instance ID", + "node_name": "Slurm Node", + "nodeset_name": "Nodeset", + "nodeset_name 1": "Nodeset", + "nvlink_instance_group": "NVLink Instance Group ID", + "state_base": "Base State" + } + } + }, + { + "id": "groupToNestedTable", + "options": { + "fields": { + "NVLink Instance Group ID": { + "aggregations": [], + "operation": "groupby" + }, + "Node": { + "aggregations": [] + }, + "Nodeset": { + "aggregations": [], + "operation": "groupby" + }, + "nodeset_name": { + "aggregations": [], + "operation": "groupby" + }, + "nvlink_instance_group": { + "aggregations": [], + "operation": "groupby" + }, + "state_base": { + "aggregations": [] + } + }, + "showSubframeHeaders": true + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "1": { + "color": "green", + "index": 0, + "text": "healthy" + }, + "2": { + "color": "red", + "index": 1, + "text": "unhealthy" + }, + "3": { + "color": "orange", + "index": 2, + "text": "mixed" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 112, + "maxPerRow": 2, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.6.14+security-04", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "WITH (\n nodes = count by (nodeset_name, node_name) (\n slurm_node_info{cluster=~\"$cluster\", \n nodeset_name=~\"$nodeset\"\n }\n ),\n\n healthy = count by (nodeset_name, node_name) (\n slurm_node_info{cluster=~\"$cluster\", \n state_base=~\"IDLE|ALLOCATED|MIXED\",\n state_is_maintenance!=\"true\",\n state_is_not_responding!=\"true\",\n nodeset_name=~\"$nodeset\"\n }\n ),\n\n total_nodes = count(nodes) by (nodeset_name),\n healthy_nodes = count(healthy) by (nodeset_name)\n)\n\n# Healthy: every node is healthy\ngroup(\n healthy_nodes == total_nodes\n) by (nodeset_name) * 1\nOR\n# Unhealthy: no healthy node exists\ngroup(\n total_nodes unless on (nodeset_name) healthy_nodes\n) by (nodeset_name) * 2\nOR\n# Mixed: some, but not all, nodes are healthy\ngroup(\n healthy_nodes < total_nodes\n) by (nodeset_name) * 3", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Nodesets Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Timeline of NodeSet health. Healthy means all nodes are IDLE or ALLOCATED and neither in maintenance nor not responding; unhealthy means none are healthy; mixed means only some are healthy.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisPlacement": "auto", + "fillOpacity": 70, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 1, + "spanNulls": false + }, + "mappings": [ + { + "options": { + "1": { + "color": "green", + "index": 0, + "text": "healthy" + }, + "2": { + "color": "red", + "index": 1, + "text": "unhealthy" + }, + "3": { + "color": "orange", + "index": 2, + "text": "mixed" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 113, + "options": { + "alignValue": "left", + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "mergeValues": true, + "rowHeight": 0.9, + "showValue": "never", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.14+security-04", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "WITH (\n nodes = count by (nodeset_name, node_name) (\n slurm_node_info{cluster=~\"$cluster\", \n nodeset_name=~\"$nodeset\"\n }\n ),\n\n healthy = count by (nodeset_name, node_name) (\n slurm_node_info{cluster=~\"$cluster\", \n state_base=~\"IDLE|ALLOCATED|MIXED\",\n state_is_maintenance!=\"true\",\n state_is_not_responding!=\"true\",\n nodeset_name=~\"$nodeset\"\n }\n ),\n\n total_nodes = count(nodes) by (nodeset_name),\n healthy_nodes = count(healthy) by (nodeset_name)\n)\n\n# Healthy: every node is healthy\ngroup(\n healthy_nodes == total_nodes\n) by (nodeset_name) * 1\nOR\n# Unhealthy: no healthy node exists\ngroup(\n total_nodes unless on (nodeset_name) healthy_nodes\n) by (nodeset_name) * 2\nOR\n# Mixed: some, but not all, nodes are healthy\ngroup(\n healthy_nodes < total_nodes\n) by (nodeset_name) * 3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{nodeset_name}}", + "range": true, + "refId": "A", + "useCustomValues": true, + "useDashValues": false + } + ], + "title": "Nodesets Status Over Time", + "type": "state-timeline" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 103, + "panels": [], + "title": "Stats", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Proportion of GPUs with at least 10% utilization to total for selected nodesets", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "light-orange", + "mode": "shades" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-orange" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 104, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.14+security-04", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", \n slurm_nodeset_name=~\"$nodeset\"\n }\n >=\n 10\n ) by(cluster)\n /\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", \n slurm_nodeset_name=~\"$nodeset\"\n }\n ) by(cluster)\n)\n *\n100", + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useCustomValues": true, + "useDashValues": false + } + ], + "title": "Nodeset Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "Only for GPUs with at least 10% utilization for selected nodesets", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "watt" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "min" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "avg" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [ + 10, + 10 + ], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "min" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 105, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.14+security-04", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(\n DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", \n slurm_nodeset_name=~\"$nodeset\"\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", \n slurm_nodeset_name=~\"$nodeset\"\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", + "hide": false, + "legendFormat": "max", + "range": true, + "refId": "C", + "useCustomValues": true, + "useDashValues": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "avg(\n DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", \n slurm_nodeset_name=~\"$nodeset\"\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", \n slurm_nodeset_name=~\"$nodeset\"\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", + "hide": false, + "legendFormat": "avg", + "range": true, + "refId": "B", + "useCustomValues": true, + "useDashValues": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "min(\n DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", \n slurm_nodeset_name=~\"$nodeset\"\n }\n and on(exported_pod,gpu)\n count(\n DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", \n slurm_nodeset_name=~\"$nodeset\"\n }\n >=\n 10\n ) by(exported_pod,gpu)\n)", + "hide": false, + "legendFormat": "min", + "range": true, + "refId": "A", + "useCustomValues": true, + "useDashValues": false + } + ], + "title": "Nodeset Power usage", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 110, + "panels": [], + "title": "Node Status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "1": { + "color": "green", + "index": 0, + "text": "healthy" + }, + "2": { + "color": "red", + "index": 1, + "text": "unhealthy" + }, + "3": { + "color": "yellow", + "index": 2, + "text": "maintenance" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#6b6b6b" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 106, + "maxPerRow": 2, + "options": { + "colorMode": "background_solid", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "11.6.14+security-04", + "repeat": "nodeset", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "# Healthy\ncount by (nodeset_name, node_name) (\n slurm_node_info{cluster=~\"$cluster\", \n state_base=~\"IDLE|ALLOCATED|MIXED\",\n state_is_maintenance!=\"true\",\n state_is_not_responding!=\"true\",\n node_name=~\"$node_name\",\n nodeset_name=~\"$nodeset\"\n }\n) * 1\nOR\n# Down\ncount by (nodeset_name, node_name) (\n slurm_node_info{cluster=~\"$cluster\", \n state_is_not_responding=\"true\",\n state_is_maintenance!=\"true\",\n node_name=~\"$node_name\",\n nodeset_name=~\"$nodeset\"\n }\n) * 2\nOR\n# Maintenance\ncount by (nodeset_name, node_name) (\n slurm_node_info{cluster=~\"$cluster\", \n state_is_maintenance=\"true\",\n node_name=~\"$node_name\",\n nodeset_name=~\"$nodeset\"\n }\n) * 3", + "hide": false, + "instant": true, + "legendFormat": "{{node_name}}", + "range": false, + "refId": "B" + } + ], + "title": "Node Status for $nodeset", + "type": "stat" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [ + "soperator", + "nodesets", + "overview" + ], + "templating": { + "list": [ + { + "current": { + "text": "VictoriaMetrics", + "value": "VictoriaMetrics" + }, + "hide": 2, + "includeAll": false, + "label": "Datasource", + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "allowCustomValue": false, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(slurm_node_info{cluster=~\"$cluster\"},nodeset_name)", + "description": "", + "includeAll": true, + "label": "Nodeset", + "multi": true, + "name": "nodeset", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(slurm_node_info{cluster=~\"$cluster\"},nodeset_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, + { + "allValue": ".*", + "allowCustomValue": false, + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(slurm_node_info{cluster=~\"$cluster\", nodeset_name=~\"$nodeset\"}, node_name)", + "description": "", + "includeAll": true, + "label": "Worker Node", + "multi": true, + "name": "node_name", + "options": [], + "query": { + "qryType": 5, + "query": "label_values(slurm_node_info{cluster=~\"$cluster\", nodeset_name=~\"$nodeset\"}, node_name)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "30s", + "1m", + "5m", + "15m", + "30m" + ] + }, + "timezone": "browser", + "title": "Nodesets Overview", + "uid": "nodesets_overview", + "version": 24 +} diff --git a/helm/soperator-monitoring-dashboards/dashboards/slurm_controller.json b/helm/soperator-monitoring-dashboards/dashboards/slurm_controller.json index 5239d0e36..14e8c2ce3 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/slurm_controller.json +++ b/helm/soperator-monitoring-dashboards/dashboards/slurm_controller.json @@ -111,7 +111,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "time() - max(container_start_time_seconds{namespace=~\"soperator\", pod=\"$pod\", container=\"slurmctld\"})", + "expr": "time() - max(container_start_time_seconds{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container=\"slurmctld\"})", "legendFormat": "slurmctld uptime", "range": true, "refId": "A" @@ -198,7 +198,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(kube_pod_status_phase{namespace=~\"soperator\", pod=\"$pod\", phase=\"Running\"})", + "expr": "max(kube_pod_status_phase{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", phase=\"Running\"})", "instant": true, "legendFormat": "phase", "range": false, @@ -292,7 +292,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (container) (\n kube_pod_container_status_restarts_total{namespace=~\"soperator\", pod=\"$pod\"}\n)", + "expr": "sum by (container) (\n kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\"}\n)", "legendFormat": "{{container}}", "range": true, "refId": "A" @@ -363,7 +363,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n changes(\n sum without (uid, pod_ip) (\n kube_pod_info{namespace=~\"soperator\", pod=\"$pod\", node!=\"\"}\n )[1d:1m]\n )\n)", + "expr": "sum(\n changes(\n sum without (uid, pod_ip) (\n kube_pod_info{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", node!=\"\"}\n )[1d:1m]\n )\n)", "instant": true, "legendFormat": "reschedules", "range": false, @@ -474,7 +474,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(slurm_controller_server_thread_count)", + "expr": "max(slurm_controller_server_thread_count{cluster=~\"$cluster\"})", "legendFormat": "threads", "range": true, "refId": "A" @@ -548,7 +548,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time((100 * sum(container_memory_working_set_bytes{namespace=\"soperator\", pod=\"$pod\", container=\"slurmctld\"}) / sum(kube_pod_container_resource_limits{namespace=\"soperator\", pod=\"$pod\", container=\"slurmctld\", resource=\"memory\"}))[1h:])", + "expr": "avg_over_time((100 * sum(container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=\"soperator\", pod=\"$pod\", container=\"slurmctld\"}) / sum(kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=\"soperator\", pod=\"$pod\", container=\"slurmctld\", resource=\"memory\"}))[1h:])", "legendFormat": "used %", "range": true, "refId": "A" @@ -622,7 +622,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time((100 * (1 - max(kubelet_volume_stats_inodes_free{persistentvolumeclaim=\"controller-spool-$pod\"} / kubelet_volume_stats_inodes{persistentvolumeclaim=\"controller-spool-$pod\"})))[1h:])", + "expr": "avg_over_time((100 * (1 - max(kubelet_volume_stats_inodes_free{cluster=~\"$cluster\", persistentvolumeclaim=\"controller-spool-$pod\"} / kubelet_volume_stats_inodes{cluster=~\"$cluster\", persistentvolumeclaim=\"controller-spool-$pod\"})))[1h:])", "legendFormat": "inodes used %", "range": true, "refId": "A" @@ -696,7 +696,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"soperator\", pod=\"$pod\", container=\"slurmctld\"}[1h]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=\"soperator\", pod=\"$pod\", container=\"slurmctld\"}[1h]))", "legendFormat": "cores", "range": true, "refId": "A" @@ -770,7 +770,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time((100 * (count(DCGM_FI_DEV_GPU_TEMP{hpc_job!=\"\"} and on(exported_pod) label_replace(slurm_node_info{is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\")) or vector(0)) / count(DCGM_FI_DEV_GPU_TEMP and on(exported_pod) label_replace(slurm_node_info{is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\")))[1h:])", + "expr": "avg_over_time((100 * (count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job!=\"\"} and on(exported_pod) label_replace(slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\")) or vector(0)) / count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\"} and on(exported_pod) label_replace(slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\")))[1h:])", "legendFormat": "allocated %", "range": true, "refId": "A" @@ -844,7 +844,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg_over_time((100 * (count(DCGM_FI_DEV_GPU_UTIL >= 10) or vector(0)) / count(DCGM_FI_DEV_GPU_UTIL))[1h:])", + "expr": "avg_over_time((100 * (count(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"} >= 10) or vector(0)) / count(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"}))[1h:])", "legendFormat": "utilized %", "range": true, "refId": "A" @@ -951,7 +951,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (device) (\n rate(container_fs_reads_bytes_total{namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}[5m])\n)", + "expr": "sum by (device) (\n rate(container_fs_reads_bytes_total{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}[5m])\n)", "legendFormat": "{{device}} read", "range": true, "refId": "A" @@ -962,7 +962,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "-1 * sum by (device) (\n rate(container_fs_writes_bytes_total{namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}[5m])\n)", + "expr": "-1 * sum by (device) (\n rate(container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}[5m])\n)", "legendFormat": "{{device}} write", "range": true, "refId": "B" @@ -1087,7 +1087,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(kubelet_volume_stats_used_bytes{persistentvolumeclaim=\"controller-spool-$pod\"})", + "expr": "max(kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", persistentvolumeclaim=\"controller-spool-$pod\"})", "legendFormat": "used", "range": true, "refId": "A" @@ -1098,7 +1098,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=\"controller-spool-$pod\"})", + "expr": "max(kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", persistentvolumeclaim=\"controller-spool-$pod\"})", "legendFormat": "capacity", "range": true, "refId": "B" @@ -1223,7 +1223,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(kubelet_volume_stats_inodes_used{persistentvolumeclaim=\"controller-spool-$pod\"})", + "expr": "max(kubelet_volume_stats_inodes_used{cluster=~\"$cluster\", persistentvolumeclaim=\"controller-spool-$pod\"})", "legendFormat": "used", "range": true, "refId": "A" @@ -1234,7 +1234,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(kubelet_volume_stats_inodes{persistentvolumeclaim=\"controller-spool-$pod\"})", + "expr": "max(kubelet_volume_stats_inodes{cluster=~\"$cluster\", persistentvolumeclaim=\"controller-spool-$pod\"})", "legendFormat": "total", "range": true, "refId": "B" @@ -1331,7 +1331,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_disk_read_time_seconds_total{instance=\"$instance\"}[$__rate_interval])\n/\nirate(node_disk_reads_completed_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_disk_read_time_seconds_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])\n/\nirate(node_disk_reads_completed_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "{{device}} - Read wait time avg", "range": true, "refId": "A" @@ -1342,7 +1342,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_disk_write_time_seconds_total{instance=\"$instance\"}[$__rate_interval])\n/\nirate(node_disk_writes_completed_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_disk_write_time_seconds_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])\n/\nirate(node_disk_writes_completed_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "{{device}} - Write wait time avg", "range": true, "refId": "B" @@ -1439,7 +1439,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_filefd_allocated{instance=\"$instance\"}", + "expr": "node_filefd_allocated{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Open files (system-wide)", "range": true, "refId": "A" @@ -1540,7 +1540,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_filesystem_readonly{instance=\"$instance\", device!~\"rootfs\", mountpoint!~\"/mnt/cloud-metadata\"}", + "expr": "node_filesystem_readonly{cluster=~\"$cluster\", instance=\"$instance\", device!~\"rootfs\", mountpoint!~\"/mnt/cloud-metadata\"}", "legendFormat": "{{mountpoint}} - ReadOnly", "range": true, "refId": "A" @@ -1551,7 +1551,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_filesystem_device_error{instance=\"$instance\", device!~\"rootfs\", fstype!~\"tmpfs\"}", + "expr": "node_filesystem_device_error{cluster=~\"$cluster\", instance=\"$instance\", device!~\"rootfs\", fstype!~\"tmpfs\"}", "legendFormat": "{{mountpoint}} - Device error", "range": true, "refId": "B" @@ -1648,7 +1648,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_disk_reads_completed_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_disk_reads_completed_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "{{device}} - reads", "range": true, "refId": "A" @@ -1659,7 +1659,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_disk_writes_completed_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_disk_writes_completed_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "{{device}} - writes", "range": true, "refId": "B" @@ -1756,7 +1756,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_disk_read_bytes_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_disk_read_bytes_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "{{device}} - read", "range": true, "refId": "A" @@ -1767,7 +1767,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "-1 * irate(node_disk_written_bytes_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "-1 * irate(node_disk_written_bytes_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "{{device}} - write", "range": true, "refId": "B" @@ -1866,7 +1866,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_disk_io_time_seconds_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_disk_io_time_seconds_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "{{device}}", "range": true, "refId": "A" @@ -1981,7 +1981,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count by (job_state) (slurm_job_info{job_state=~\"PENDING|RUNNING|SUSPENDED|COMPLETING|CONFIGURING|RESIZING|REQUEUED|SIGNALING|STAGE_OUT\"}) or label_replace(vector(0), \"job_state\", \"PENDING\", \"\", \"\")", + "expr": "count by (job_state) (slurm_job_info{cluster=~\"$cluster\", job_state=~\"PENDING|RUNNING|SUSPENDED|COMPLETING|CONFIGURING|RESIZING|REQUEUED|SIGNALING|STAGE_OUT\"}) or label_replace(vector(0), \"job_state\", \"PENDING\", \"\", \"\")", "legendFormat": "{{job_state}}", "range": true, "refId": "A" @@ -2081,7 +2081,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * (count(DCGM_FI_DEV_GPU_TEMP{hpc_job!=\"\"} and on(exported_pod) label_replace(slurm_node_info{is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\")) or vector(0)) / count(DCGM_FI_DEV_GPU_TEMP and on(exported_pod) label_replace(slurm_node_info{is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\"))", + "expr": "100 * (count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", hpc_job!=\"\"} and on(exported_pod) label_replace(slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\")) or vector(0)) / count(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\"} and on(exported_pod) label_replace(slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"}, \"exported_pod\", \"$1\", \"node_name\", \"(.*)\"))", "legendFormat": "allocated", "range": true, "refId": "A" @@ -2181,7 +2181,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * (count(DCGM_FI_DEV_GPU_UTIL >= 10) or vector(0)) / count(DCGM_FI_DEV_GPU_UTIL)", + "expr": "100 * (count(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"} >= 10) or vector(0)) / count(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\"})", "legendFormat": "utilized", "range": true, "refId": "A" @@ -2391,7 +2391,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=~\"DOWN|ERROR\", state_is_maintenance!=\"true\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=~\"DOWN|ERROR\", state_is_maintenance!=\"true\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Down", @@ -2405,7 +2405,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=\"IDLE\", state_is_maintenance!=\"true\", state_is_drain=\"true\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=\"IDLE\", state_is_maintenance!=\"true\", state_is_drain=\"true\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Drained", @@ -2419,7 +2419,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=~\"ALLOCATED|MIXED\", state_is_maintenance!=\"true\", state_is_drain=\"true\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=~\"ALLOCATED|MIXED\", state_is_maintenance!=\"true\", state_is_drain=\"true\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Draining", @@ -2433,7 +2433,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=~\"ALLOCATED|MIXED\", state_is_maintenance!=\"true\", state_is_drain!=\"true\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=~\"ALLOCATED|MIXED\", state_is_maintenance!=\"true\", state_is_drain!=\"true\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Busy", @@ -2447,7 +2447,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=\"IDLE\", state_is_maintenance!=\"true\", state_is_drain!=\"true\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=\"IDLE\", state_is_maintenance!=\"true\", state_is_drain!=\"true\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Idle", @@ -2461,7 +2461,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_is_maintenance=\"true\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_is_maintenance=\"true\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Maintenance", @@ -2475,7 +2475,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(slurm_node_info{state_base=\"UNKNOWN\", state_is_maintenance!=\"true\"})", + "expr": "count(slurm_node_info{cluster=~\"$cluster\", state_base=\"UNKNOWN\", state_is_maintenance!=\"true\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Unknown", @@ -2579,7 +2579,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * (count(slurm_node_info{state_base=~\"ALLOCATED|MIXED\"}) or vector(0)) / count(slurm_node_info{is_unavailable=\"\"})", + "expr": "100 * (count(slurm_node_info{cluster=~\"$cluster\", state_base=~\"ALLOCATED|MIXED\"}) or vector(0)) / count(slurm_node_info{cluster=~\"$cluster\", is_unavailable=\"\"})", "legendFormat": "allocated", "range": true, "refId": "A" @@ -2689,7 +2689,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (message_type) (\n rate(slurm_controller_rpc_calls_total[5m])\n)", + "expr": "sum by (message_type) (\n rate(slurm_controller_rpc_calls_total{cluster=~\"$cluster\"}[5m])\n)", "legendFormat": "{{message_type}}", "range": true, "refId": "A" @@ -2786,7 +2786,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (message_type) (rate(slurm_controller_rpc_duration_seconds_total[5m]))\n/\nsum by (message_type) (rate(slurm_controller_rpc_calls_total[5m]))", + "expr": "sum by (message_type) (rate(slurm_controller_rpc_duration_seconds_total{cluster=~\"$cluster\"}[5m]))\n/\nsum by (message_type) (rate(slurm_controller_rpc_calls_total{cluster=~\"$cluster\"}[5m]))", "legendFormat": "{{message_type}}", "range": true, "refId": "A" @@ -2883,7 +2883,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (message_type) (rate(slurm_controller_rpc_duration_seconds_total[5m]))", + "expr": "sum by (message_type) (rate(slurm_controller_rpc_duration_seconds_total{cluster=~\"$cluster\"}[5m]))", "legendFormat": "{{message_type}}", "range": true, "refId": "A" @@ -2980,7 +2980,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "topk(10,\n sum by (user) (\n rate(slurm_controller_rpc_user_calls_total[5m])\n )\n)", + "expr": "topk(10,\n sum by (user) (\n rate(slurm_controller_rpc_user_calls_total{cluster=~\"$cluster\"}[5m])\n )\n)", "legendFormat": "{{user}}", "range": true, "refId": "A" @@ -3077,7 +3077,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "topk(10,\n sum by (user) (\n rate(slurm_controller_rpc_user_duration_seconds_total[5m])\n )\n)", + "expr": "topk(10,\n sum by (user) (\n rate(slurm_controller_rpc_user_duration_seconds_total{cluster=~\"$cluster\"}[5m])\n )\n)", "legendFormat": "{{user}}", "range": true, "refId": "A" @@ -3174,7 +3174,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "topk(10,\n sum by (user) (rate(slurm_controller_rpc_user_duration_seconds_total[5m]))\n /\n sum by (user) (rate(slurm_controller_rpc_user_calls_total[5m]))\n)", + "expr": "topk(10,\n sum by (user) (rate(slurm_controller_rpc_user_duration_seconds_total{cluster=~\"$cluster\"}[5m]))\n /\n sum by (user) (rate(slurm_controller_rpc_user_calls_total{cluster=~\"$cluster\"}[5m]))\n)", "legendFormat": "{{user}}", "range": true, "refId": "A" @@ -3271,7 +3271,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_agent_queue_size", + "expr": "slurm_agent_queue_size{cluster=~\"$cluster\"}", "legendFormat": "queued", "range": true, "refId": "A" @@ -3282,7 +3282,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_agent_thread_cnt", + "expr": "slurm_agent_thread_cnt{cluster=~\"$cluster\"}", "legendFormat": "dispatching", "range": true, "refId": "B" @@ -3293,7 +3293,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_agent_cnt", + "expr": "slurm_agent_cnt{cluster=~\"$cluster\"}", "legendFormat": "agent pool", "range": true, "refId": "C" @@ -3390,7 +3390,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_slurmdbd_queue_size", + "expr": "slurm_slurmdbd_queue_size{cluster=~\"$cluster\"}", "legendFormat": "buffered messages", "range": true, "refId": "A" @@ -3513,7 +3513,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_schedule_cycle_last / 1e6", + "expr": "slurm_schedule_cycle_last{cluster=~\"$cluster\"} / 1e6", "legendFormat": "last cycle (1 sample / scrape, default 30s)", "range": true, "refId": "A" @@ -3610,7 +3610,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_schedule_cycle_cnt[5m]) * 60", + "expr": "rate(slurm_schedule_cycle_cnt{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "cycles/min", "range": true, "refId": "A" @@ -3621,7 +3621,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_schedule_queue_len", + "expr": "slurm_schedule_queue_len{cluster=~\"$cluster\"}", "legendFormat": "queue length", "range": true, "refId": "B" @@ -3718,7 +3718,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_sched_exit_end[5m]) * 60", + "expr": "rate(slurm_sched_exit_end{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "End of job queue", "range": true, "refId": "A" @@ -3729,7 +3729,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_sched_exit_max_depth[5m]) * 60", + "expr": "rate(slurm_sched_exit_max_depth{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "Hit default_queue_depth", "range": true, "refId": "B" @@ -3740,7 +3740,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_sched_exit_max_job_start[5m]) * 60", + "expr": "rate(slurm_sched_exit_max_job_start{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "Hit sched_max_job_start", "range": true, "refId": "C" @@ -3751,7 +3751,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_sched_exit_lic[5m]) * 60", + "expr": "rate(slurm_sched_exit_lic{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "Blocked on licenses", "range": true, "refId": "D" @@ -3762,7 +3762,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_sched_exit_rpc_cnt[5m]) * 60", + "expr": "rate(slurm_sched_exit_rpc_cnt{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "Hit max_rpc_cnt", "range": true, "refId": "E" @@ -3773,7 +3773,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_sched_exit_timeout[5m]) * 60", + "expr": "rate(slurm_sched_exit_timeout{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "Timeout (max_sched_time)", "range": true, "refId": "F" @@ -3875,7 +3875,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_bf_active", + "expr": "slurm_bf_active{cluster=~\"$cluster\"}", "legendFormat": "state", "range": true, "refId": "A" @@ -3980,7 +3980,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "time() - slurm_bf_when_last_cycle", + "expr": "time() - slurm_bf_when_last_cycle{cluster=~\"$cluster\"}", "legendFormat": "age", "range": true, "refId": "A" @@ -4085,7 +4085,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_bf_cycle_last / 1e6", + "expr": "slurm_bf_cycle_last{cluster=~\"$cluster\"} / 1e6", "legendFormat": "last cycle (1 sample / scrape, default 30s)", "range": true, "refId": "A" @@ -4182,7 +4182,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_bf_cycle_cnt[5m]) * 60", + "expr": "rate(slurm_bf_cycle_cnt{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "cycles/min", "range": true, "refId": "A" @@ -4279,7 +4279,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_bf_queue_len", + "expr": "slurm_bf_queue_len{cluster=~\"$cluster\"}", "legendFormat": "backfill queue length", "range": true, "refId": "A" @@ -4376,7 +4376,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_bf_last_depth", + "expr": "slurm_bf_last_depth{cluster=~\"$cluster\"}", "legendFormat": "examined (slurm_bf_last_depth)", "range": true, "refId": "A" @@ -4387,7 +4387,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_bf_last_depth_try", + "expr": "slurm_bf_last_depth_try{cluster=~\"$cluster\"}", "legendFormat": "attempted to start (slurm_bf_last_depth_try)", "range": true, "refId": "B" @@ -4484,7 +4484,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "slurm_bf_table_size", + "expr": "slurm_bf_table_size{cluster=~\"$cluster\"}", "legendFormat": "time-slot table size", "range": true, "refId": "A" @@ -4581,7 +4581,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_bf_exit_end[5m]) * 60", + "expr": "rate(slurm_bf_exit_end{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "End of job queue", "range": true, "refId": "A" @@ -4592,7 +4592,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_bf_exit_max_job_start[5m]) * 60", + "expr": "rate(slurm_bf_exit_max_job_start{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "Hit bf_max_job_start", "range": true, "refId": "B" @@ -4603,7 +4603,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_bf_exit_max_job_test[5m]) * 60", + "expr": "rate(slurm_bf_exit_max_job_test{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "Hit bf_max_job_test", "range": true, "refId": "C" @@ -4614,7 +4614,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_bf_exit_state_changed[5m]) * 60", + "expr": "rate(slurm_bf_exit_state_changed{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "System state changed", "range": true, "refId": "D" @@ -4625,7 +4625,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_bf_exit_table_limit[5m]) * 60", + "expr": "rate(slurm_bf_exit_table_limit{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "Hit table size limit (bf_node_space_size)", "range": true, "refId": "E" @@ -4636,7 +4636,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(slurm_bf_exit_timeout[5m]) * 60", + "expr": "rate(slurm_bf_exit_timeout{cluster=~\"$cluster\"}[5m]) * 60", "legendFormat": "Timeout (bf_max_time)", "range": true, "refId": "F" @@ -4782,7 +4782,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (container) (\n container_memory_working_set_bytes{namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}\n)", + "expr": "sum by (container) (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}\n)", "legendFormat": "{{container}}", "range": true, "refId": "A" @@ -4793,7 +4793,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_MemAvailable_bytes{instance=\"$instance\"}", + "expr": "node_memory_MemAvailable_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "MemAvailable", "range": true, "refId": "B" @@ -4891,7 +4891,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100\n* sum by (container) (\n container_memory_working_set_bytes{namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}\n )\n/ on (container)\n (\n sum by (container) (\n kube_pod_container_resource_limits{namespace=~\"soperator\", pod=\"$pod\", resource=\"memory\"}\n )\n or\n on (container)\n sum by (container) (\n container_memory_working_set_bytes{namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"} * 0\n )\n + on () group_left scalar(node_memory_MemAvailable_bytes{instance=\"$instance\"})\n )", + "expr": "100\n* sum by (container) (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}\n )\n/ on (container)\n (\n sum by (container) (\n kube_pod_container_resource_limits{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", resource=\"memory\"}\n )\n or\n on (container)\n sum by (container) (\n container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"} * 0\n )\n + on () group_left scalar(node_memory_MemAvailable_bytes{cluster=~\"$cluster\", instance=\"$instance\"})\n )", "legendFormat": "{{container}}", "range": true, "refId": "A" @@ -4988,7 +4988,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_Active_bytes{instance=\"$instance\"}", + "expr": "node_memory_Active_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", "range": true, "refId": "A" @@ -4999,7 +4999,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_Inactive_bytes{instance=\"$instance\"}", + "expr": "node_memory_Inactive_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", "range": true, "refId": "B" @@ -5126,7 +5126,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_Committed_AS_bytes{instance=\"$instance\"}", + "expr": "node_memory_Committed_AS_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", "range": true, "refId": "A" @@ -5137,7 +5137,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_CommitLimit_bytes{instance=\"$instance\"}", + "expr": "node_memory_CommitLimit_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", "range": true, "refId": "B" @@ -5234,7 +5234,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_Writeback_bytes{instance=\"$instance\"}", + "expr": "node_memory_Writeback_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Writeback - Memory which is actively being written back to disk", "range": true, "refId": "A" @@ -5245,7 +5245,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_WritebackTmp_bytes{instance=\"$instance\"}", + "expr": "node_memory_WritebackTmp_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", "range": true, "refId": "B" @@ -5256,7 +5256,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_Dirty_bytes{instance=\"$instance\"}", + "expr": "node_memory_Dirty_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", "range": true, "refId": "C" @@ -5353,7 +5353,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_vmstat_pgfault{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_vmstat_pgfault{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "Pgfault - Page major and minor fault operations", "range": true, "refId": "A" @@ -5364,7 +5364,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_vmstat_pgmajfault{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_vmstat_pgmajfault{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "Pgmajfault - Major page fault operations", "range": true, "refId": "B" @@ -5464,7 +5464,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_vmstat_oom_kill{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_vmstat_oom_kill{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "oom killer invocations", "range": true, "refId": "A" @@ -5561,7 +5561,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_vmstat_pgpgin{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_vmstat_pgpgin{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "Pagesin - Page in operations", "range": true, "refId": "A" @@ -5572,7 +5572,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "-1 * irate(node_vmstat_pgpgout{instance=\"$instance\"}[$__rate_interval])", + "expr": "-1 * irate(node_vmstat_pgpgout{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "Pagesout - Page out operations", "range": true, "refId": "B" @@ -5669,7 +5669,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_Mapped_bytes{instance=\"$instance\"}", + "expr": "node_memory_Mapped_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries", "range": true, "refId": "A" @@ -5680,7 +5680,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_Shmem_bytes{instance=\"$instance\"}", + "expr": "node_memory_Shmem_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", "range": true, "refId": "B" @@ -5691,7 +5691,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_ShmemHugePages_bytes{instance=\"$instance\"}", + "expr": "node_memory_ShmemHugePages_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages", "range": true, "refId": "C" @@ -5702,7 +5702,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_ShmemPmdMapped_bytes{instance=\"$instance\"}", + "expr": "node_memory_ShmemPmdMapped_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "ShmemPmdMapped - Amount of shared (shmem/tmpfs) memory backed by huge pages", "range": true, "refId": "D" @@ -5799,7 +5799,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_AnonHugePages_bytes{instance=\"$instance\"}", + "expr": "node_memory_AnonHugePages_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "AnonHugePages - Memory in anonymous huge pages", "range": true, "refId": "A" @@ -5810,7 +5810,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_memory_AnonPages_bytes{instance=\"$instance\"}", + "expr": "node_memory_AnonPages_bytes{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "AnonPages - Memory in user pages not backed by files", "range": true, "refId": "B" @@ -5907,7 +5907,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "container_memory_rss{namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}", + "expr": "container_memory_rss{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}", "legendFormat": "{{container}} rss", "range": true, "refId": "A" @@ -5918,7 +5918,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "container_memory_cache{namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}", + "expr": "container_memory_cache{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}", "legendFormat": "{{container}} cache", "range": true, "refId": "B" @@ -5929,7 +5929,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "container_memory_mapped_file{namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}", + "expr": "container_memory_mapped_file{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}", "legendFormat": "{{container}} mapped_file", "range": true, "refId": "C" @@ -6041,7 +6041,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(container_network_receive_bytes_total{namespace=~\"soperator\", pod=\"$pod\"}[5m])\n)", + "expr": "sum(\n rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\"}[5m])\n)", "legendFormat": "rx", "range": true, "refId": "A" @@ -6052,7 +6052,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "-1 * sum(\n rate(container_network_transmit_bytes_total{namespace=~\"soperator\", pod=\"$pod\"}[5m])\n)", + "expr": "-1 * sum(\n rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\"}[5m])\n)", "legendFormat": "tx", "range": true, "refId": "B" @@ -6149,7 +6149,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_receive_packets_total{instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", + "expr": "irate(node_network_receive_packets_total{cluster=~\"$cluster\", instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", "legendFormat": "{{device}} - Receive", "range": true, "refId": "A" @@ -6160,7 +6160,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "-1 * irate(node_network_transmit_packets_total{instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", + "expr": "-1 * irate(node_network_transmit_packets_total{cluster=~\"$cluster\", instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", "legendFormat": "{{device}} - Transmit", "range": true, "refId": "B" @@ -6257,7 +6257,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_receive_errs_total{instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", + "expr": "irate(node_network_receive_errs_total{cluster=~\"$cluster\", instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", "legendFormat": "{{device}} - Receive errors", "range": true, "refId": "A" @@ -6268,7 +6268,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_transmit_errs_total{instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", + "expr": "irate(node_network_transmit_errs_total{cluster=~\"$cluster\", instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", "legendFormat": "{{device}} - Transmit errors", "range": true, "refId": "B" @@ -6279,7 +6279,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_receive_drop_total{instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", + "expr": "irate(node_network_receive_drop_total{cluster=~\"$cluster\", instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", "legendFormat": "{{device}} - Receive drops", "range": true, "refId": "C" @@ -6290,7 +6290,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_transmit_drop_total{instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", + "expr": "irate(node_network_transmit_drop_total{cluster=~\"$cluster\", instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}[$__rate_interval])", "legendFormat": "{{device}} - Transmit drops", "range": true, "refId": "D" @@ -6387,7 +6387,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_netstat_Tcp_CurrEstab{instance=\"$instance\"}", + "expr": "node_netstat_Tcp_CurrEstab{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "CurrEstab - TCP connections in ESTABLISHED or CLOSE-WAIT", "range": true, "refId": "A" @@ -6398,7 +6398,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_netstat_Tcp_ActiveOpens{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "ActiveOpens/s - new outbound connections", "range": true, "refId": "B" @@ -6409,7 +6409,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_netstat_Tcp_PassiveOpens{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "PassiveOpens/s - new inbound connections", "range": true, "refId": "C" @@ -6506,7 +6506,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_Tcp_RetransSegs{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_netstat_Tcp_RetransSegs{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "RetransSegs - TCP segments retransmitted", "range": true, "refId": "A" @@ -6517,7 +6517,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "SynRetrans - SYN/ACK retransmits", "range": true, "refId": "B" @@ -6528,7 +6528,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_netstat_TcpExt_ListenDrops{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored (overflow)", "range": true, "refId": "C" @@ -6539,7 +6539,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_netstat_TcpExt_ListenOverflows{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "ListenOverflows - listen queue overflowed", "range": true, "refId": "D" @@ -6550,7 +6550,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_Tcp_OutRsts{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_netstat_Tcp_OutRsts{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "OutRsts - segments sent with RST flag", "range": true, "refId": "E" @@ -6676,7 +6676,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_nf_conntrack_entries{instance=\"$instance\"}", + "expr": "node_nf_conntrack_entries{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "entries", "range": true, "refId": "A" @@ -6687,7 +6687,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_nf_conntrack_entries_limit{instance=\"$instance\"}", + "expr": "node_nf_conntrack_entries_limit{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "limit", "range": true, "refId": "B" @@ -6785,7 +6785,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_network_up{instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}", + "expr": "node_network_up{cluster=~\"$cluster\", instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}", "legendFormat": "{{device}} - operstate=up", "range": true, "refId": "A" @@ -6796,7 +6796,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_network_carrier{instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}", + "expr": "node_network_carrier{cluster=~\"$cluster\", instance=\"$instance\", device=~\"eth.*|en.*|bond.*\"}", "legendFormat": "{{device}} - physical link carrier", "range": true, "refId": "B" @@ -6911,7 +6911,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_receive_bytes_total{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", + "expr": "irate(node_network_receive_bytes_total{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", "legendFormat": "{{pod}} {{device}} RX", "range": true, "refId": "A" @@ -6922,7 +6922,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "-1 * irate(node_network_transmit_bytes_total{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", + "expr": "-1 * irate(node_network_transmit_bytes_total{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", "legendFormat": "{{pod}} {{device}} TX", "range": true, "refId": "B" @@ -7019,7 +7019,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_netstat_Tcp_CurrEstab{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\"}", + "expr": "node_netstat_Tcp_CurrEstab{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\"}", "legendFormat": "{{pod}} established", "range": true, "refId": "A" @@ -7116,7 +7116,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_receive_packets_total{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", + "expr": "irate(node_network_receive_packets_total{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", "legendFormat": "{{pod}} {{device}} RX", "range": true, "refId": "A" @@ -7127,7 +7127,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "-1 * irate(node_network_transmit_packets_total{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", + "expr": "-1 * irate(node_network_transmit_packets_total{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", "legendFormat": "{{pod}} {{device}} TX", "range": true, "refId": "B" @@ -7224,7 +7224,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_Tcp_RetransSegs{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\"}[$__rate_interval])", + "expr": "irate(node_netstat_Tcp_RetransSegs{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\"}[$__rate_interval])", "legendFormat": "{{pod}} retrans", "range": true, "refId": "A" @@ -7235,7 +7235,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\"}[$__rate_interval])", + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\"}[$__rate_interval])", "legendFormat": "{{pod}} SYN retrans", "range": true, "refId": "B" @@ -7332,7 +7332,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_receive_errs_total{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", + "expr": "irate(node_network_receive_errs_total{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", "legendFormat": "{{device}} RX errs", "range": true, "refId": "A" @@ -7343,7 +7343,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_receive_drop_total{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", + "expr": "irate(node_network_receive_drop_total{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", "legendFormat": "{{device}} RX drops", "range": true, "refId": "B" @@ -7354,7 +7354,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_transmit_errs_total{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", + "expr": "irate(node_network_transmit_errs_total{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", "legendFormat": "{{device}} TX errs", "range": true, "refId": "C" @@ -7365,7 +7365,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_network_transmit_drop_total{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", + "expr": "irate(node_network_transmit_drop_total{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\", device!=\"lo\"}[$__rate_interval])", "legendFormat": "{{device}} TX drops", "range": true, "refId": "D" @@ -7462,7 +7462,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_TcpExt_ListenDrops{pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\"}[$__rate_interval])", + "expr": "irate(node_netstat_TcpExt_ListenDrops{cluster=~\"$cluster\", pod=~\"controller-.*\", job=\"soperator/custom-node-exporter\"}[$__rate_interval])", "legendFormat": "{{pod}} listen drops", "range": true, "refId": "A" @@ -7574,7 +7574,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (container) (\n rate(container_cpu_usage_seconds_total{namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}[5m])\n)", + "expr": "sum by (container) (\n rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\", container!=\"\", container!=\"POD\"}[5m])\n)", "legendFormat": "{{container}}", "range": true, "refId": "A" @@ -7670,7 +7670,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$instance\", mode=\"system\"}[$__rate_interval]))", + "expr": "sum(irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=\"$instance\", mode=\"system\"}[$__rate_interval]))", "legendFormat": "System - Processes executing in kernel mode", "range": true, "refId": "A" @@ -7681,7 +7681,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$instance\", mode=\"user\"}[$__rate_interval]))", + "expr": "sum(irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=\"$instance\", mode=\"user\"}[$__rate_interval]))", "legendFormat": "User - Normal processes executing in user mode", "range": true, "refId": "B" @@ -7692,7 +7692,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$instance\", mode=\"nice\"}[$__rate_interval]))", + "expr": "sum(irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=\"$instance\", mode=\"nice\"}[$__rate_interval]))", "legendFormat": "Nice - Niced processes in user mode", "range": true, "refId": "C" @@ -7703,7 +7703,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$instance\", mode=\"iowait\"}[$__rate_interval]))", + "expr": "sum(irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=\"$instance\", mode=\"iowait\"}[$__rate_interval]))", "legendFormat": "Iowait - Waiting for I/O to complete", "range": true, "refId": "D" @@ -7714,7 +7714,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$instance\", mode=\"irq\"}[$__rate_interval]))", + "expr": "sum(irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=\"$instance\", mode=\"irq\"}[$__rate_interval]))", "legendFormat": "Irq - Servicing interrupts", "range": true, "refId": "E" @@ -7725,7 +7725,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$instance\", mode=\"softirq\"}[$__rate_interval]))", + "expr": "sum(irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=\"$instance\", mode=\"softirq\"}[$__rate_interval]))", "legendFormat": "Softirq - Servicing softirqs", "range": true, "refId": "F" @@ -7736,7 +7736,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$instance\", mode=\"steal\"}[$__rate_interval]))", + "expr": "sum(irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=\"$instance\", mode=\"steal\"}[$__rate_interval]))", "legendFormat": "Steal - Time stolen by the hypervisor", "range": true, "refId": "G" @@ -7747,7 +7747,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(irate(node_cpu_seconds_total{instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", + "expr": "sum(irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=\"$instance\", mode=\"idle\"}[$__rate_interval]))", "legendFormat": "Idle - Waiting for something to happen", "range": true, "refId": "H" @@ -7844,7 +7844,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_load1{instance=\"$instance\"}", + "expr": "node_load1{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Load 1m", "range": true, "refId": "A" @@ -7855,7 +7855,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_load5{instance=\"$instance\"}", + "expr": "node_load5{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Load 5m", "range": true, "refId": "B" @@ -7866,7 +7866,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "node_load15{instance=\"$instance\"}", + "expr": "node_load15{cluster=~\"$cluster\", instance=\"$instance\"}", "legendFormat": "Load 15m", "range": true, "refId": "C" @@ -7963,7 +7963,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_context_switches_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_context_switches_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "Context switches/s", "range": true, "refId": "A" @@ -7974,7 +7974,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_intr_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "irate(node_intr_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "Interrupts/s", "range": true, "refId": "B" @@ -8100,7 +8100,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * max(kube_node_status_condition{node=\"$node\", condition=\"MemoryPressure\", status=\"true\"})", + "expr": "100 * max(kube_node_status_condition{cluster=~\"$cluster\", node=\"$node\", condition=\"MemoryPressure\", status=\"true\"})", "legendFormat": "Kubelet MemoryPressure (0/100)", "range": true, "refId": "A" @@ -8111,7 +8111,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * irate(node_pressure_memory_waiting_seconds_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "100 * irate(node_pressure_memory_waiting_seconds_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "Memory PSI some - % time at least one task stalled on memory", "range": true, "refId": "B" @@ -8122,7 +8122,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * irate(node_pressure_memory_stalled_seconds_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "100 * irate(node_pressure_memory_stalled_seconds_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "Memory PSI full - % time ALL tasks stalled on memory", "range": true, "refId": "C" @@ -8133,7 +8133,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * irate(node_pressure_cpu_waiting_seconds_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "100 * irate(node_pressure_cpu_waiting_seconds_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "CPU PSI some - % time at least one task stalled on CPU", "range": true, "refId": "D" @@ -8144,7 +8144,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * irate(node_pressure_io_waiting_seconds_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "100 * irate(node_pressure_io_waiting_seconds_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "I/O PSI some - % time at least one task stalled on I/O", "range": true, "refId": "E" @@ -8155,7 +8155,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "100 * irate(node_pressure_io_stalled_seconds_total{instance=\"$instance\"}[$__rate_interval])", + "expr": "100 * irate(node_pressure_io_stalled_seconds_total{cluster=~\"$cluster\", instance=\"$instance\"}[$__rate_interval])", "legendFormat": "I/O PSI full - % time ALL tasks stalled on I/O", "range": true, "refId": "F" @@ -8197,6 +8197,46 @@ "skipUrlSync": false, "type": "datasource" }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, { "allowCustomValue": false, "current": { @@ -8207,7 +8247,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(kube_pod_info{namespace=~\"soperator\", pod=~\"controller-[0-9]+\"}, pod)", + "definition": "label_values(kube_pod_info{cluster=~\"$cluster\", namespace=~\"soperator\", pod=~\"controller-[0-9]+\"}, pod)", "description": "Controller pod to inspect. Discovered from kube_pod_info; defaults to controller-0.", "includeAll": false, "label": "Controller Pod", @@ -8216,7 +8256,7 @@ "options": [], "query": { "qryType": 5, - "query": "label_values(kube_pod_info{namespace=~\"soperator\", pod=~\"controller-[0-9]+\"}, pod)", + "query": "label_values(kube_pod_info{cluster=~\"$cluster\", namespace=~\"soperator\", pod=~\"controller-[0-9]+\"}, pod)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -8234,7 +8274,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(kube_pod_info{namespace=~\"soperator\", pod=\"$pod\"}, node)", + "definition": "label_values(kube_pod_info{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\"}, node)", "description": "K8s node hosting the selected controller pod. Hidden; used for joining node-exporter metrics.", "hide": 2, "includeAll": false, @@ -8244,7 +8284,7 @@ "options": [], "query": { "qryType": 5, - "query": "label_values(kube_pod_info{namespace=~\"soperator\", pod=\"$pod\"}, node)", + "query": "label_values(kube_pod_info{cluster=~\"$cluster\", namespace=~\"soperator\", pod=\"$pod\"}, node)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, @@ -8262,7 +8302,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(node_uname_info{nodename=\"$node\"}, instance)", + "definition": "label_values(node_uname_info{cluster=~\"$cluster\", nodename=\"$node\"}, instance)", "description": "node-exporter `instance` label for the controller node. Hidden; used to filter node_* metrics.", "hide": 2, "includeAll": false, @@ -8272,7 +8312,7 @@ "options": [], "query": { "qryType": 5, - "query": "label_values(node_uname_info{nodename=\"$node\"}, instance)", + "query": "label_values(node_uname_info{cluster=~\"$cluster\", nodename=\"$node\"}, instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, @@ -8303,4 +8343,4 @@ "uid": "soperator-slurm-controller", "version": 1, "weekStart": "" -} \ No newline at end of file +} diff --git a/helm/soperator-monitoring-dashboards/dashboards/workers_detailed_stats.json b/helm/soperator-monitoring-dashboards/dashboards/workers_detailed_stats.json index 168069d7d..f0f65674c 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/workers_detailed_stats.json +++ b/helm/soperator-monitoring-dashboards/dashboards/workers_detailed_stats.json @@ -26,7 +26,7 @@ "uid": "${datasource}" }, "enable": false, - "expr": "group by (node, pod) (\n changes(\n sum without (uid, pod_ip)(\n kube_pod_info{pod=~\"worker-.*\", node!=\"\"}\n )\n ) > 0\n and on (pod) \n sum by (pod)(\n kube_pod_info{pod=~\"worker-.*\", node!=\"\"}\n )\n)", + "expr": "group by (node, pod) (\n changes(\n sum without (uid, pod_ip)(\n kube_pod_info{cluster=~\"$cluster\", pod=~\"worker-.*\", node!=\"\"}\n )\n ) > 0\n and on (pod) \n sum by (pod)(\n kube_pod_info{cluster=~\"$cluster\", pod=~\"worker-.*\", node!=\"\"}\n )\n)", "filter": { "exclude": true, "ids": [324] @@ -110,7 +110,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "group by (node) (kube_pod_info{pod=\"$worker\",node!=\"\"})", + "expr": "group by (node) (kube_pod_info{cluster=~\"$cluster\", pod=\"$worker\",node!=\"\"})", "instant": false, "legendFormat": "__auto", "range": true, @@ -186,7 +186,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_SM_CLOCK{Hostname=~\"$node\"}*1000000)", + "expr": "avg(DCGM_FI_DEV_SM_CLOCK{cluster=~\"$cluster\", Hostname=~\"$node\"}*1000000)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -264,7 +264,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_MEM_CLOCK{Hostname=~\"$node\"}*1000000)", + "expr": "avg(DCGM_FI_DEV_MEM_CLOCK{cluster=~\"$cluster\", Hostname=~\"$node\"}*1000000)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -378,7 +378,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (gpu) (\n DCGM_FI_DEV_FB_USED{Hostname=~\"$node\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{Hostname=~\"$node\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{Hostname=~\"$node\", exported_pod=~\"$worker\"})\n)", + "expr": "sum by (gpu) (\n DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"})\n)", "format": "time_series", "hide": false, "interval": "", @@ -481,7 +481,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(gpu) (DCGM_FI_DEV_GPU_UTIL{Hostname=~\"$node\", exported_pod=~\"$worker\"})", + "expr": "sum by(gpu) (DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -583,7 +583,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "1e6*sum(DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL{Hostname=~\"$node\", exported_pod=~\"$worker\"})", + "expr": "1e6*sum(DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -686,7 +686,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(gpu) (DCGM_FI_DEV_MEM_COPY_UTIL{Hostname=~\"$node\", exported_pod=~\"$worker\"})", + "expr": "sum by(gpu) (DCGM_FI_DEV_MEM_COPY_UTIL{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -787,7 +787,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(gpu) (DCGM_FI_DEV_POWER_USAGE{Hostname=~\"$node\", exported_pod=~\"$worker\"})", + "expr": "sum by(gpu) (DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -888,7 +888,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(gpu) (DCGM_FI_DEV_GPU_TEMP{Hostname=~\"$node\", exported_pod=~\"$worker\"})", + "expr": "sum by(gpu) (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -1002,7 +1002,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_PROF_PCIE_TX_BYTES{Hostname=~\"$node\", exported_pod=~\"$worker\"})", + "expr": "sum(DCGM_FI_PROF_PCIE_TX_BYTES{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -1017,7 +1017,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_PROF_PCIE_RX_BYTES{Hostname=~\"$node\", exported_pod=~\"$worker\"})", + "expr": "sum(DCGM_FI_PROF_PCIE_RX_BYTES{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -1131,7 +1131,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(node_infiniband_port_data_transmitted_bytes_total{instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n)\n", + "expr": "sum(\n rate(node_infiniband_port_data_transmitted_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n)\n", "format": "time_series", "hide": false, "interval": "", @@ -1146,7 +1146,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n rate(node_infiniband_port_data_received_bytes_total{instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", + "expr": "sum(\n rate(node_infiniband_port_data_received_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n)", "format": "time_series", "hide": false, "instant": false, @@ -1247,7 +1247,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(gpu) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$node\", exported_pod=~\"$worker\"} * 100)", + "expr": "sum by(gpu) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"} * 100)", "format": "time_series", "hide": false, "interval": "", @@ -1348,7 +1348,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(gpu) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$node\", exported_pod=~\"$worker\"} * 1979)", + "expr": "sum by(gpu) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"} * 1979)", "format": "time_series", "hide": false, "interval": "", @@ -1449,7 +1449,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(gpu) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{Hostname=~\"$node\", exported_pod=~\"$worker\"} * 989.7)", + "expr": "sum by(gpu) (DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", Hostname=~\"$node\", exported_pod=~\"$worker\"} * 989.7)", "format": "time_series", "hide": false, "interval": "", @@ -1685,7 +1685,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{instance=~\"$ip\", mode=\"system\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"system\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1700,7 +1700,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{instance=~\"$ip\", mode=\"user\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"user\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "User - Normal processes executing in user mode", @@ -1714,7 +1714,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{instance=~\"$ip\", mode=\"nice\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"nice\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Nice - Niced processes executing in user mode", @@ -1728,7 +1728,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{instance=~\"$ip\", mode=\"iowait\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"iowait\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Iowait - Waiting for I/O to complete", @@ -1742,7 +1742,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{instance=~\"$ip\", mode=\"irq\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"irq\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Irq - Servicing interrupts", @@ -1756,7 +1756,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{instance=~\"$ip\", mode=\"softirq\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"softirq\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Softirq - Servicing softirqs", @@ -1770,7 +1770,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{instance=~\"$ip\", mode=\"steal\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"steal\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Steal - Time spent in other operating systems when running in a virtualized environment", @@ -1784,7 +1784,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(\n irate(node_cpu_seconds_total{instance=~\"$ip\", mode=\"idle\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{instance=~\"$ip\"}) by (cpu)))", + "expr": "sum(\n irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"idle\"}[$__rate_interval])\n * on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n * on(nodename) group_left(exported_pod) clamp(count by (nodename, hpc_job) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)\n) / scalar(count(count(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}) by (cpu)))", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2190,7 +2190,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(node_memory_MemTotal_bytes{instance=~\"$ip\"} - node_memory_MemFree_bytes{instance=~\"$ip\"} - node_memory_Buffers_bytes{instance=~\"$ip\"} - node_memory_Cached_bytes{instance=~\"$ip\"} - node_memory_Slab_bytes{instance=~\"$ip\"} - node_memory_PageTables_bytes{instance=~\"$ip\"} - node_memory_SwapCached_bytes{instance=~\"$ip\"})\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "(node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"} - node_memory_MemFree_bytes{cluster=~\"$cluster\", instance=~\"$ip\"} - node_memory_Buffers_bytes{cluster=~\"$cluster\", instance=~\"$ip\"} - node_memory_Cached_bytes{cluster=~\"$cluster\", instance=~\"$ip\"} - node_memory_Slab_bytes{cluster=~\"$cluster\", instance=~\"$ip\"} - node_memory_PageTables_bytes{cluster=~\"$cluster\", instance=~\"$ip\"} - node_memory_SwapCached_bytes{cluster=~\"$cluster\", instance=~\"$ip\"})\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2204,7 +2204,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_PageTables_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_PageTables_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2217,7 +2217,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_SwapCached_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_SwapCached_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "SwapCache - Memory that keeps track of pages that have been fetched from swap but not yet been modified", @@ -2229,7 +2229,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Slab_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Slab_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2242,7 +2242,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Cached_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Cached_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2255,7 +2255,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Buffers_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Buffers_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2268,7 +2268,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_MemFree_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_MemFree_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2281,7 +2281,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "(node_memory_SwapTotal_bytes{instance=~\"$ip\"} - node_memory_SwapFree_bytes{instance=~\"$ip\"})\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "(node_memory_SwapTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"} - node_memory_SwapFree_bytes{cluster=~\"$cluster\", instance=~\"$ip\"})\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2294,7 +2294,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_HardwareCorrupted_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_HardwareCorrupted_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -2467,7 +2467,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_network_receive_bytes_total{instance=~\"$ip\"}[$__rate_interval])*8\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_network_receive_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])*8\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} - Receive", @@ -2479,7 +2479,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_network_transmit_bytes_total{instance=~\"$ip\"}[$__rate_interval])*8\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_network_transmit_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])*8\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} - Transmit", @@ -2581,7 +2581,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(node_filesystem_size_bytes{instance=~\"$ip\",device!~'rootfs'} - node_filesystem_avail_bytes{instance=~\"$ip\",device!~'rootfs'})\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "(node_filesystem_size_bytes{cluster=~\"$cluster\", instance=~\"$ip\",device!~'rootfs'} - node_filesystem_avail_bytes{cluster=~\"$cluster\", instance=~\"$ip\",device!~'rootfs'})\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{mountpoint}}", @@ -3010,7 +3010,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_reads_completed_total{instance=~\"$ip\",device=~\"$diskdevices\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_reads_completed_total{cluster=~\"$cluster\", instance=~\"$ip\",device=~\"$diskdevices\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "intervalFactor": 4, "legendFormat": "{{device}} - Reads completed", "refId": "A", @@ -3021,7 +3021,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_writes_completed_total{instance=~\"$ip\",device=~\"$diskdevices\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_writes_completed_total{cluster=~\"$cluster\", instance=~\"$ip\",device=~\"$diskdevices\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "intervalFactor": 1, "legendFormat": "{{device}} - Writes completed", "refId": "B", @@ -3235,7 +3235,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_read_bytes_total{instance=~\"$ip\",device=~\"$diskdevices\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_read_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\",device=~\"$diskdevices\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -3248,7 +3248,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_written_bytes_total{instance=~\"$ip\",device=~\"$diskdevices\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_written_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\",device=~\"$diskdevices\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -3378,7 +3378,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$ip\",device=~\"$diskdevices\"} [$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_io_time_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\",device=~\"$diskdevices\"} [$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "interval": "", @@ -3511,7 +3511,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=~\"$ip\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=~\"$ip\"}[1m])))", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"user\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[1m])))", "hide": false, "legendFormat": "Guest - Time spent running a virtual CPU for a guest operating system", "range": true, @@ -3523,7 +3523,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{instance=~\"$ip\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{instance=~\"$ip\"}[1m])))", + "expr": "sum by(instance) (irate(node_cpu_guest_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\", mode=\"nice\"}[1m])) / on(instance) group_left sum by (instance)((irate(node_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[1m])))", "hide": false, "legendFormat": "GuestNice - Time spent running a niced guest (virtual CPU for guest operating system)", "range": true, @@ -3893,7 +3893,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Inactive_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Inactive_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Inactive - Memory which has been less recently used. It is more eligible to be reclaimed for other purposes", @@ -3905,7 +3905,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Active_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Active_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Active - Memory that has been used more recently and usually not reclaimed unless absolutely necessary", @@ -4281,7 +4281,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Committed_AS_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Committed_AS_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Committed_AS - Amount of memory presently allocated on the system", @@ -4293,7 +4293,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_CommitLimit_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_CommitLimit_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "CommitLimit - Amount of memory currently available to be allocated on the system", @@ -4650,7 +4650,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Inactive_file_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Inactive_file_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4663,7 +4663,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Inactive_anon_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Inactive_anon_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4676,7 +4676,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Active_file_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Active_file_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -4689,7 +4689,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Active_anon_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Active_anon_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -5076,7 +5076,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Writeback_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Writeback_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Writeback - Memory which is actively being written back to disk", @@ -5088,7 +5088,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_WritebackTmp_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_WritebackTmp_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "WritebackTmp - Memory used by FUSE for temporary writeback buffers", @@ -5100,7 +5100,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Dirty_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Dirty_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Dirty - Memory which is waiting to get written back to the disk", @@ -5481,7 +5481,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Mapped_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Mapped_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Mapped - Used memory in mapped pages files which have been mapped, such as libraries", @@ -5493,7 +5493,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Shmem_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Shmem_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Shmem - Used shared memory (shared between several processes, thus including RAM disks)", @@ -5505,7 +5505,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_ShmemHugePages_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_ShmemHugePages_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -5518,7 +5518,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_ShmemPmdMapped_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_ShmemPmdMapped_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -5876,7 +5876,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_KernelStack_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_KernelStack_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "KernelStack - Kernel memory stack. This is not reclaimable", @@ -5888,7 +5888,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_Percpu_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_Percpu_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -6260,7 +6260,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_VmallocChunk_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_VmallocChunk_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -6273,7 +6273,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_VmallocTotal_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_VmallocTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -6286,7 +6286,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_VmallocUsed_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_VmallocUsed_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -6670,7 +6670,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_AnonHugePages_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_AnonHugePages_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "AnonHugePages - Memory in anonymous huge pages", @@ -6682,7 +6682,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_AnonPages_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_AnonPages_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "AnonPages - Memory in user pages not backed by files", @@ -7068,7 +7068,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_memory_NFS_Unstable_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_memory_NFS_Unstable_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "NFS Unstable - Memory in NFS pages sent to the server, but not yet committed to the storage", @@ -7454,7 +7454,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_vmstat_oom_kill{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_vmstat_oom_kill{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -7567,7 +7567,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_vmstat_pgpgin{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_vmstat_pgpgin{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pagesin - Page in operations", @@ -7579,7 +7579,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_vmstat_pgpgout{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_vmstat_pgpgout{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pagesout - Page out operations", @@ -7955,7 +7955,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_vmstat_pgfault{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_vmstat_pgfault{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pgfault - Page major and minor fault operations", @@ -7967,7 +7967,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_vmstat_pgmajfault{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_vmstat_pgmajfault{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pgmajfault - Major page fault operations", @@ -7979,7 +7979,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_vmstat_pgfault{instance=~\"$ip\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_vmstat_pgfault{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval]) - irate(node_vmstat_pgmajfault{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Pgminfault - Minor page fault operations", @@ -8406,7 +8406,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_reads_completed_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_reads_completed_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "intervalFactor": 4, "legendFormat": "{{device}} - Reads completed", "refId": "A", @@ -8417,7 +8417,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_writes_completed_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_writes_completed_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "intervalFactor": 1, "legendFormat": "{{device}} - Writes completed", "refId": "B", @@ -8829,7 +8829,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_read_bytes_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_read_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 4, "legendFormat": "{{device}} - Read bytes", @@ -8841,7 +8841,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_written_bytes_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_written_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} - Written bytes", @@ -9254,7 +9254,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_read_time_seconds_total{instance=~\"$ip\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_read_time_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval]) / irate(node_disk_reads_completed_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "hide": false, "interval": "", "intervalFactor": 4, @@ -9267,7 +9267,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_write_time_seconds_total{instance=~\"$ip\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_write_time_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval]) / irate(node_disk_writes_completed_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "hide": false, "interval": "", "intervalFactor": 1, @@ -9670,7 +9670,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_io_time_weighted_seconds_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_io_time_weighted_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "intervalFactor": 4, "legendFormat": "{{device}}", @@ -10083,7 +10083,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_reads_merged_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_reads_merged_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "intervalFactor": 1, "legendFormat": "{{device}} - Read merged", "refId": "A", @@ -10094,7 +10094,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_writes_merged_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_writes_merged_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "intervalFactor": 1, "legendFormat": "{{device}} - Write merged", "refId": "B", @@ -10495,7 +10495,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_io_time_seconds_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_io_time_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "intervalFactor": 4, "legendFormat": "{{device}} - IO", @@ -10507,7 +10507,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_discard_time_seconds_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_discard_time_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "intervalFactor": 4, "legendFormat": "{{device}} - discard", @@ -10909,7 +10909,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_disk_io_now{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_disk_io_now{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "intervalFactor": 4, "legendFormat": "{{device}} - IO now", @@ -11310,7 +11310,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_discards_completed_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_discards_completed_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "intervalFactor": 4, "legendFormat": "{{device}} - Discards completed", @@ -11322,7 +11322,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_disk_discards_merged_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_disk_discards_merged_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "intervalFactor": 1, "legendFormat": "{{device}} - Discards merged", @@ -11437,7 +11437,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_filesystem_avail_bytes{instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_filesystem_avail_bytes{cluster=~\"$cluster\", instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -11451,7 +11451,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_filesystem_free_bytes{instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_filesystem_free_bytes{cluster=~\"$cluster\", instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -11464,7 +11464,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_filesystem_size_bytes{instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_filesystem_size_bytes{cluster=~\"$cluster\", instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": true, "intervalFactor": 1, @@ -11566,7 +11566,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_filesystem_files_free{instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_filesystem_files_free{cluster=~\"$cluster\", instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -11668,7 +11668,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_filefd_maximum{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_filefd_maximum{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 4, "legendFormat": "Max open files", @@ -11680,7 +11680,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_filefd_allocated{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_filefd_allocated{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Open files", @@ -11781,7 +11781,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_filesystem_files{instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_filesystem_files{cluster=~\"$cluster\", instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -11900,7 +11900,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_filesystem_readonly{instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_filesystem_readonly{cluster=~\"$cluster\", instance=~\"$ip\",device!~'rootfs'}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{mountpoint}} - ReadOnly", @@ -11912,7 +11912,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_filesystem_device_error{instance=~\"$ip\",device!~'rootfs',fstype!~'tmpfs'}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_filesystem_device_error{cluster=~\"$cluster\", instance=~\"$ip\",device!~'rootfs',fstype!~'tmpfs'}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -12100,7 +12100,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_network_receive_packets_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_network_receive_packets_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -12113,7 +12113,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_network_transmit_packets_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_network_transmit_packets_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -12227,7 +12227,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_network_receive_errs_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_network_receive_errs_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} - Receive errors", @@ -12239,7 +12239,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_network_transmit_errs_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_network_transmit_errs_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} - Transmit errors", @@ -12352,7 +12352,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_network_receive_drop_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_network_receive_drop_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} - Receive drop", @@ -12364,7 +12364,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_network_transmit_drop_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_network_transmit_drop_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{device}} - Transmit drop", @@ -12484,7 +12484,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_nf_conntrack_entries{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_nf_conntrack_entries{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "NF conntrack entries", @@ -12496,7 +12496,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_nf_conntrack_entries_limit{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_nf_conntrack_entries_limit{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "NF conntrack limit", @@ -12597,7 +12597,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_network_speed_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_network_speed_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{ device }} - Speed", @@ -12697,7 +12697,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_network_up{operstate=\"up\",instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_network_up{cluster=~\"$cluster\", operstate=\"up\",instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{interface}} - Operational state UP", @@ -12709,7 +12709,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_network_carrier{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_network_carrier{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "instant": false, "legendFormat": "{{device}} - Physical link state", @@ -12847,7 +12847,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Udp_InDatagrams{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Udp_InDatagrams{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -12860,7 +12860,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Udp_OutDatagrams{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Udp_OutDatagrams{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -12960,7 +12960,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Udp_InErrors{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Udp_InErrors{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -12973,7 +12973,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Udp_NoPorts{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Udp_NoPorts{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -12986,7 +12986,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_UdpLite_InErrors{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_UdpLite_InErrors{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "legendFormat": "InErrors Lite - UDPLite Datagrams that could not be delivered to an application", "refId": "C" @@ -12996,7 +12996,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Udp_RcvbufErrors{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Udp_RcvbufErrors{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -13009,7 +13009,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Udp_SndbufErrors{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Udp_SndbufErrors{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -13134,7 +13134,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Tcp_InSegs{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Tcp_InSegs{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "instant": false, "interval": "", @@ -13148,7 +13148,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Tcp_OutSegs{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Tcp_OutSegs{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -13250,7 +13250,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_TcpExt_ListenOverflows{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_TcpExt_ListenOverflows{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "interval": "", @@ -13264,7 +13264,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_TcpExt_ListenDrops{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_TcpExt_ListenDrops{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "interval": "", @@ -13278,7 +13278,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_TcpExt_TCPSynRetrans{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -13291,7 +13291,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Tcp_RetransSegs{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Tcp_RetransSegs{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", "refId": "D" @@ -13301,7 +13301,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Tcp_InErrs{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Tcp_InErrs{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", "refId": "E" @@ -13311,7 +13311,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Tcp_OutRsts{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Tcp_OutRsts{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "interval": "", "legendFormat": "OutRsts - Segments sent with RST flag", "refId": "F" @@ -13322,7 +13322,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_TcpExt_TCPRcvQDrop{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_TcpExt_TCPRcvQDrop{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "hide": false, "interval": "", "legendFormat": "TCPRcvQDrop - Packets meant to be queued in rcv queue but dropped because socket rcvbuf limit hit", @@ -13335,7 +13335,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "irate(node_netstat_TcpExt_TCPOFOQueue{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_TcpExt_TCPOFOQueue{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "hide": false, "interval": "", "legendFormat": "TCPOFOQueue - TCP layer receives an out of order packet and has enough memory to queue it", @@ -13455,7 +13455,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_netstat_Tcp_CurrEstab{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_netstat_Tcp_CurrEstab{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "interval": "", @@ -13469,7 +13469,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_netstat_Tcp_MaxConn{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_netstat_Tcp_MaxConn{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "interval": "", @@ -13571,7 +13571,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Tcp_ActiveOpens{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Tcp_ActiveOpens{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -13584,7 +13584,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_netstat_Tcp_PassiveOpens{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_netstat_Tcp_PassiveOpens{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -13699,7 +13699,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_context_switches_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_context_switches_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Context switches", @@ -13711,7 +13711,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_intr_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_intr_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -13812,7 +13812,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_load1{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_load1{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 4, "legendFormat": "Load 1m", @@ -13824,7 +13824,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_load5{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_load5{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 4, "legendFormat": "Load 5m", @@ -13836,7 +13836,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_load15{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_load15{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 4, "legendFormat": "Load 15m", @@ -13936,7 +13936,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_entropy_available_bits{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_entropy_available_bits{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "intervalFactor": 1, "legendFormat": "Entropy available to random number generators", @@ -14099,7 +14099,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_pressure_cpu_waiting_seconds_total{instance=~\"$ip\"}[$__rate_interval])", + "expr": "rate(node_pressure_cpu_waiting_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])", "format": "time_series", "intervalFactor": 1, "legendFormat": "CPU some", @@ -14113,7 +14113,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_pressure_memory_waiting_seconds_total{instance=~\"$ip\"}[$__rate_interval])", + "expr": "rate(node_pressure_memory_waiting_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -14128,7 +14128,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_pressure_memory_stalled_seconds_total{instance=~\"$ip\"}[$__rate_interval])", + "expr": "rate(node_pressure_memory_stalled_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -14143,7 +14143,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_pressure_io_waiting_seconds_total{instance=~\"$ip\"}[$__rate_interval])", + "expr": "rate(node_pressure_io_waiting_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -14158,7 +14158,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "rate(node_pressure_io_stalled_seconds_total{instance=~\"$ip\"}[$__rate_interval])", + "expr": "rate(node_pressure_io_stalled_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -14280,7 +14280,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "process_max_fds{instance=~\"$ip\"}", + "expr": "process_max_fds{cluster=~\"$cluster\", instance=~\"$ip\"}", "interval": "", "intervalFactor": 1, "legendFormat": "Maximum open file descriptors", @@ -14292,7 +14292,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "process_open_fds{instance=~\"$ip\"}", + "expr": "process_open_fds{cluster=~\"$cluster\", instance=~\"$ip\"}", "interval": "", "intervalFactor": 1, "legendFormat": "Open file descriptors", @@ -14391,7 +14391,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(node_schedstat_timeslices_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(node_schedstat_timeslices_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -14491,7 +14491,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "irate(process_cpu_seconds_total{instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "irate(process_cpu_seconds_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__rate_interval])\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -14622,7 +14622,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_timex_estimated_error_seconds{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_timex_estimated_error_seconds{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "interval": "", @@ -14636,7 +14636,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_timex_offset_seconds{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_timex_offset_seconds{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "interval": "", @@ -14650,7 +14650,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_timex_maxerror_seconds{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_timex_maxerror_seconds{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "hide": false, "interval": "", @@ -14752,7 +14752,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_timex_loop_time_constant{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_timex_loop_time_constant{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -14869,7 +14869,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_timex_sync_status{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_timex_sync_status{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -14882,7 +14882,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_timex_frequency_adjustment_ratio{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_timex_frequency_adjustment_ratio{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -14983,7 +14983,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_timex_tick_seconds{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_timex_tick_seconds{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -14996,7 +14996,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "node_timex_tai_offset_seconds{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", + "expr": "node_timex_tai_offset_seconds{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(exported_pod) clamp(count by (nodename, exported_pod) (label_move(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"}, \"Hostname\", \"nodename\")), 1, 1)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -15037,6 +15037,46 @@ "skipUrlSync": false, "type": "datasource" }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, { "current": { "text": "worker-0", @@ -15046,7 +15086,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(kube_pod_info,pod)", + "definition": "label_values(kube_pod_info{cluster=~\"$cluster\"},pod)", "description": "Select one worker to check it's detailed stats in panels.", "includeAll": false, "label": "Worker", @@ -15054,7 +15094,7 @@ "options": [], "query": { "qryType": 1, - "query": "label_values(kube_pod_info,pod)", + "query": "label_values(kube_pod_info{cluster=~\"$cluster\"},pod)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -15070,16 +15110,17 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(kube_pod_info{pod=~\"$worker\"},node)", + "definition": "label_values(kube_pod_info{cluster=~\"$cluster\", pod=~\"$worker\"},node)", "hide": 2, "includeAll": true, + "allValue": ".*", "label": "node", "multi": true, "name": "node", "options": [], "query": { "qryType": 1, - "query": "label_values(kube_pod_info{pod=~\"$worker\"},node)", + "query": "label_values(kube_pod_info{cluster=~\"$cluster\", pod=~\"$worker\"},node)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, @@ -15114,15 +15155,16 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(node_uname_info{nodename=~\"$node\"},instance)", + "definition": "label_values(node_uname_info{cluster=~\"$cluster\", nodename=~\"$node\"},instance)", "hide": 2, "includeAll": true, + "allValue": ".*", "multi": true, "name": "ip", "options": [], "query": { "qryType": 1, - "query": "label_values(node_uname_info{nodename=~\"$node\"},instance)", + "query": "label_values(node_uname_info{cluster=~\"$cluster\", nodename=~\"$node\"},instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, diff --git a/helm/soperator-monitoring-dashboards/dashboards/workers_overview.json b/helm/soperator-monitoring-dashboards/dashboards/workers_overview.json index 1e6ec2f6d..c94b3a322 100644 --- a/helm/soperator-monitoring-dashboards/dashboards/workers_overview.json +++ b/helm/soperator-monitoring-dashboards/dashboards/workers_overview.json @@ -25,7 +25,7 @@ "uid": "${datasource}" }, "enable": false, - "expr": "group by (node, pod) (\n changes(\n sum without (uid, pod_ip)(\n kube_pod_info{pod=~\"worker-.*\", node!=\"\"}\n )\n ) > 0\n and on (pod) \n sum by (pod)(\n kube_pod_info{pod=~\"worker-.*\", node!=\"\"}\n )\n)", + "expr": "group by (node, pod) (\n changes(\n sum without (uid, pod_ip)(\n kube_pod_info{cluster=~\"$cluster\", pod=~\"worker-.*\", node!=\"\"}\n )\n ) > 0\n and on (pod) \n sum by (pod)(\n kube_pod_info{cluster=~\"$cluster\", pod=~\"worker-.*\", node!=\"\"}\n )\n)", "filter": { "exclude": true, "ids": [328] @@ -161,7 +161,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg (\n 1 - rate(node_cpu_seconds_total{mode=\"idle\"}[1m])\n) * 100", + "expr": "avg (\n 1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\"}[1m])\n) * 100", "instant": false, "legendFormat": "__auto", "range": true, @@ -244,7 +244,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n (\n node_memory_MemTotal_bytes - \n node_memory_MemFree_bytes - \n node_memory_Buffers_bytes - \n node_memory_Cached_bytes\n ) / \n node_memory_MemTotal_bytes{instance=~\"$ip\"}\n) * 100", + "expr": "avg(\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n ) / \n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n) * 100", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -329,7 +329,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum(node_filesystem_size_bytes{device!=\"rootfs\",instance=~\"$ip\"}) - \n sum(node_filesystem_avail_bytes{device!=\"rootfs\",instance=~\"$ip\"})\n) / \nsum(node_filesystem_size_bytes{device!=\"rootfs\",instance=~\"$ip\"})", + "expr": "(\n sum(node_filesystem_size_bytes{cluster=~\"$cluster\", device!=\"rootfs\",instance=~\"$ip\"}) - \n sum(node_filesystem_avail_bytes{cluster=~\"$cluster\", device!=\"rootfs\",instance=~\"$ip\"})\n) / \nsum(node_filesystem_size_bytes{cluster=~\"$cluster\", device!=\"rootfs\",instance=~\"$ip\"})", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -418,7 +418,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\"})", + "expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -505,7 +505,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_DEV_POWER_USAGE{exported_pod=~\"$worker\"})", + "expr": "sum(DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -590,7 +590,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"})", + "expr": "avg(DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -675,7 +675,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_MEM_COPY_UTIL{exported_pod=~\"$worker\"})", + "expr": "avg(DCGM_FI_DEV_MEM_COPY_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -753,7 +753,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_SM_CLOCK{exported_pod=~\"$worker\"}*1000000)", + "expr": "avg(DCGM_FI_DEV_SM_CLOCK{cluster=~\"$cluster\", exported_pod=~\"$worker\"}*1000000)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -831,7 +831,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_MEM_CLOCK{exported_pod=~\"$worker\"}*1000000)", + "expr": "avg(DCGM_FI_DEV_MEM_CLOCK{cluster=~\"$cluster\", exported_pod=~\"$worker\"}*1000000)", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -1101,7 +1101,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sort_by_label(\n group by (node, pod) (\n kube_pod_info{pod=~\"$worker\",node!=\"\"}\n ),\n \"pod\", \"node\"\n)", + "expr": "sort_by_label(\n group by (node, pod) (\n kube_pod_info{cluster=~\"$cluster\", pod=~\"$worker\",node!=\"\"}\n ),\n \"pod\", \"node\"\n)", "format": "time_series", "instant": false, "interval": "", @@ -1264,7 +1264,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\"})", + "expr": "max(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -1279,7 +1279,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\"})", + "expr": "avg(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -1294,7 +1294,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\"})", + "expr": "min(DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -1453,7 +1453,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{exported_pod=~\"$worker\"}))", + "expr": "max(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\"}))", "format": "time_series", "hide": false, "interval": "", @@ -1468,7 +1468,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{exported_pod=~\"$worker\"}))", + "expr": "avg(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\"}))", "format": "time_series", "hide": false, "interval": "", @@ -1483,7 +1483,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{exported_pod=~\"$worker\"}))", + "expr": "min(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\"}))", "format": "time_series", "hide": false, "interval": "", @@ -1651,7 +1651,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max (DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"})", + "expr": "max (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -1666,7 +1666,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg (DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"})", + "expr": "avg (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -1681,7 +1681,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min (DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"})", + "expr": "min (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -1831,7 +1831,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(1 - rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))", + "expr": "max(1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))", "instant": false, "legendFormat": "max", "range": true, @@ -1843,7 +1843,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(1 - rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))", + "expr": "avg(1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "avg", @@ -1856,7 +1856,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(1 - rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))", + "expr": "min(1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "min", @@ -1953,7 +1953,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{exported_pod=~\"$worker\"} * 1979)", + "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", exported_pod=~\"$worker\"} * 1979)", "format": "time_series", "hide": false, "interval": "", @@ -1968,7 +1968,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{exported_pod=~\"$worker\"} * 989.7)", + "expr": "sum(DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{cluster=~\"$cluster\", exported_pod=~\"$worker\"} * 989.7)", "format": "time_series", "hide": false, "interval": "", @@ -2105,7 +2105,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (rate(node_infiniband_port_data_transmitted_bytes_total{instance=~\"$ip\"}[$__interval]))", + "expr": "sum (rate(node_infiniband_port_data_transmitted_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval]))", "format": "time_series", "hide": false, "interval": "", @@ -2120,7 +2120,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (rate(node_infiniband_port_data_received_bytes_total{instance=~\"$ip\"}[$__interval]))", + "expr": "sum (rate(node_infiniband_port_data_received_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval]))", "format": "time_series", "hide": false, "instant": false, @@ -2278,7 +2278,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "max(\n (\n node_memory_MemTotal_bytes - \n node_memory_MemFree_bytes - \n node_memory_Buffers_bytes - \n node_memory_Cached_bytes\n )\n /\n node_memory_MemTotal_bytes{instance=~\"$ip\"}\n)", + "expr": "max(\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n)", "format": "time_series", "hide": false, "instant": false, @@ -2293,7 +2293,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(\n (\n node_memory_MemTotal_bytes - \n node_memory_MemFree_bytes - \n node_memory_Buffers_bytes - \n node_memory_Cached_bytes\n )\n /\n node_memory_MemTotal_bytes{instance=~\"$ip\"}\n)", + "expr": "avg(\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n)", "format": "time_series", "hide": false, "instant": false, @@ -2308,7 +2308,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "min(\n (\n node_memory_MemTotal_bytes - \n node_memory_MemFree_bytes - \n node_memory_Buffers_bytes - \n node_memory_Cached_bytes\n )\n /\n node_memory_MemTotal_bytes{instance=~\"$ip\"}\n)", + "expr": "min(\n (\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n )\n /\n node_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n)", "format": "time_series", "hide": false, "instant": false, @@ -2424,7 +2424,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_UTIL{exported_pod=~\"$worker\"})", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -2527,7 +2527,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{exported_pod=~\"$worker\"}))", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}/(DCGM_FI_DEV_FB_USED{cluster=~\"$cluster\", exported_pod=~\"$worker\"}+DCGM_FI_DEV_FB_FREE{cluster=~\"$cluster\", exported_pod=~\"$worker\"}))", "format": "time_series", "hide": false, "interval": "", @@ -2632,7 +2632,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_MEM_COPY_UTIL{exported_pod=~\"$worker\"})", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_MEM_COPY_UTIL{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -2730,7 +2730,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (exported_pod) (DCGM_FI_DEV_POWER_USAGE{exported_pod=~\"$worker\"})", + "expr": "sum by (exported_pod) (DCGM_FI_DEV_POWER_USAGE{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -2831,7 +2831,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_TEMP{exported_pod=~\"$worker\"})", + "expr": "avg by (exported_pod) (DCGM_FI_DEV_GPU_TEMP{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -2945,7 +2945,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (err_msg, exported_pod) (rate(DCGM_FI_DEV_XID_ERRORS{exported_pod=~\"$worker\"}))", + "expr": "sum by (err_msg, exported_pod) (rate(DCGM_FI_DEV_XID_ERRORS{cluster=~\"$cluster\", exported_pod=~\"$worker\"}))", "instant": false, "legendFormat": "{{exported_pod}}: {{err_msg}}", "range": true, @@ -3052,7 +3052,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg(1 - rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$ip\"}[$__rate_interval])) by (instance)\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "avg(1 - rate(node_cpu_seconds_total{cluster=~\"$cluster\", mode=\"idle\", instance=~\"$ip\"}[$__rate_interval])) by (instance)\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "instant": false, "legendFormat": "{{pod}}", "range": true, @@ -3149,7 +3149,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n node_memory_MemTotal_bytes - \n node_memory_MemFree_bytes - \n node_memory_Buffers_bytes - \n node_memory_Cached_bytes\n)\n/\nnode_memory_MemTotal_bytes{instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "(\n node_memory_MemTotal_bytes{cluster=~\"$cluster\"} - \n node_memory_MemFree_bytes{cluster=~\"$cluster\"} - \n node_memory_Buffers_bytes{cluster=~\"$cluster\"} - \n node_memory_Cached_bytes{cluster=~\"$cluster\"}\n)\n/\nnode_memory_MemTotal_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "format": "time_series", "hide": false, "instant": false, @@ -3268,7 +3268,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (instance) (rate(node_disk_read_bytes_total{instance=~\"$ip\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "sum by (instance) (rate(node_disk_read_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "format": "time_series", "hide": false, "instant": false, @@ -3283,7 +3283,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (instance) (rate(node_disk_written_bytes_total{instance=~\"$ip\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "sum by (instance) (rate(node_disk_written_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "format": "time_series", "hide": false, "instant": false, @@ -3384,7 +3384,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum by (instance, device) (node_filesystem_size_bytes{instance=~\"$ip\"}) - \n sum by (instance, device) (node_filesystem_avail_bytes{instance=~\"$ip\"})\n) / \nsum by (instance, device) (node_filesystem_size_bytes{instance=~\"$ip\",device!~\"none|shm|tmpfs|cloud-metadata|accounting|.*vda15\"})\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "(\n sum by (instance, device) (node_filesystem_size_bytes{cluster=~\"$cluster\", instance=~\"$ip\"}) - \n sum by (instance, device) (node_filesystem_avail_bytes{cluster=~\"$cluster\", instance=~\"$ip\"})\n) / \nsum by (instance, device) (node_filesystem_size_bytes{cluster=~\"$cluster\", instance=~\"$ip\",device!~\"none|shm|tmpfs|cloud-metadata|accounting|.*vda15\"})\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "hide": false, "instant": false, "legendFormat": "{{device}} {{pod}}", @@ -3487,7 +3487,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "avg by (instance, device) (\n 1 - \n (\n node_filesystem_files_free{instance=~\"$ip\",device!~'none|shm|tmpfs|cloud-metadata|accounting|.*vda15'}\n /\n node_filesystem_files{instance=~\"$ip\",device!~'none|shm|tmpfs|cloud-metadata|accounting|.*vda15'}\n )\n)\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "avg by (instance, device) (\n 1 - \n (\n node_filesystem_files_free{cluster=~\"$cluster\", instance=~\"$ip\",device!~'none|shm|tmpfs|cloud-metadata|accounting|.*vda15'}\n /\n node_filesystem_files{cluster=~\"$cluster\", instance=~\"$ip\",device!~'none|shm|tmpfs|cloud-metadata|accounting|.*vda15'}\n )\n)\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -3633,7 +3633,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{instance=~\"$ip\",device=~\"eth.*\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "sum by (instance) (rate(node_network_transmit_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\",device=~\"eth.*\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "format": "time_series", "hide": false, "interval": ".5m", @@ -3648,7 +3648,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (instance) (rate(node_network_receive_bytes_total{instance=~\"$ip\",device=~\"eth.*\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "sum by (instance) (rate(node_network_receive_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\",device=~\"eth.*\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "format": "time_series", "hide": false, "instant": false, @@ -3784,7 +3784,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (instance) (rate(node_infiniband_port_data_transmitted_bytes_total{instance=~\"$ip\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "sum by (instance) (rate(node_infiniband_port_data_transmitted_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "format": "time_series", "hide": false, "interval": "", @@ -3799,7 +3799,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (instance) (rate(node_infiniband_port_data_received_bytes_total{instance=~\"$ip\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "sum by (instance) (rate(node_infiniband_port_data_received_bytes_total{cluster=~\"$cluster\", instance=~\"$ip\"}[$__interval]))\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "format": "time_series", "hide": false, "instant": false, @@ -3918,7 +3918,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (exported_pod) (DCGM_FI_PROF_PCIE_TX_BYTES{exported_pod=~\"$worker\"})", + "expr": "sum by (exported_pod) (DCGM_FI_PROF_PCIE_TX_BYTES{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -3933,7 +3933,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (exported_pod) (DCGM_FI_PROF_PCIE_RX_BYTES{exported_pod=~\"$worker\"})", + "expr": "sum by (exported_pod) (DCGM_FI_PROF_PCIE_RX_BYTES{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -4032,7 +4032,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "1e6*sum by (exported_pod) (DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL{exported_pod=~\"$worker\"})", + "expr": "1e6*sum by (exported_pod) (DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL{cluster=~\"$cluster\", exported_pod=~\"$worker\"})", "format": "time_series", "hide": false, "interval": "", @@ -4133,7 +4133,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (instance) (\n rate({__name__=~\"node_infiniband_.*(error|downed|discards|dropped).*\", instance=~\"$ip\"}[$__interval])\n)\n* on(instance) group_left(nodename) node_uname_info\n* on(nodename) group_left(pod) label_move(kube_pod_info{pod=~\"worker.*\"}, \"node\", \"nodename\")", + "expr": "sum by (instance) (\n rate({cluster=~\"$cluster\", __name__=~\"node_infiniband_.*(error|downed|discards|dropped).*\", instance=~\"$ip\"}[$__interval])\n)\n* on(instance) group_left(nodename) node_uname_info{cluster=~\"$cluster\"}\n* on(nodename) group_left(pod) label_move(kube_pod_info{cluster=~\"$cluster\", pod=~\"worker.*\"}, \"node\", \"nodename\")", "format": "time_series", "hide": false, "interval": "", @@ -4234,7 +4234,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (error) (\n rate(\n label_copy(\n {__name__=~\"node_infiniband_.*(error|downed|discards|dropped).*\", instance=~\".*\"},\n \"__name__\",\n \"error\",\n )[$__interval]\n )\n)\n", + "expr": "sum by (error) (\n rate(\n label_copy(\n {cluster=~\"$cluster\", __name__=~\"node_infiniband_.*(error|downed|discards|dropped).*\", instance=~\".*\"},\n \"__name__\",\n \"error\",\n )[$__interval]\n )\n)\n", "hide": false, "instant": false, "legendFormat": "__auto", @@ -4274,6 +4274,46 @@ "skipUrlSync": false, "type": "datasource" }, + { + "baseFilters": [], + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "filters": [], + "hide": 2, + "label": "O11y", + "name": "o11y", + "type": "adhoc" + }, + { + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(node_os_info, cluster)", + "hide": 2, + "includeAll": true, + "label": "Cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(node_os_info, cluster)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "sort": 1, + "type": "query" + }, { "allowCustomValue": false, "current": { @@ -4284,16 +4324,17 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(kube_node_labels{label_nebius_com_gpu_name=~\"GB.*\"},label_slurm_nebius_ai_nodeset_name)", + "definition": "label_values(kube_node_labels{cluster=~\"$cluster\", label_nebius_com_gpu_name=~\"GB.*\"},label_slurm_nebius_ai_nodeset_name)", "description": "For GB platforms", "includeAll": true, + "allValue": ".*", "label": "Rack", "multi": true, "name": "rack", "options": [], "query": { "qryType": 1, - "query": "label_values(kube_node_labels{label_nebius_com_gpu_name=~\"GB.*\"},label_slurm_nebius_ai_nodeset_name)", + "query": "label_values(kube_node_labels{cluster=~\"$cluster\", label_nebius_com_gpu_name=~\"GB.*\"},label_slurm_nebius_ai_nodeset_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -4301,7 +4342,7 @@ "type": "query" }, { - "allValue": "", + "allValue": ".*", "allowCustomValue": false, "current": { "text": ["worker-rack0-0"], @@ -4311,7 +4352,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(slurm_node_info{node_name=~\"$rack-.*\"}, node_name)", + "definition": "label_values(slurm_node_info{cluster=~\"$cluster\", node_name=~\"$rack-.*\"}, node_name)", "description": "Select as many workers as you need to check their aggregate and separate stats in panels.", "includeAll": true, "label": "Workers", @@ -4320,7 +4361,7 @@ "options": [], "query": { "qryType": 5, - "query": "label_values(slurm_node_info{node_name=~\"$rack-.*\"}, node_name)", + "query": "label_values(slurm_node_info{cluster=~\"$cluster\", node_name=~\"$rack-.*\"}, node_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, @@ -4337,16 +4378,17 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(kube_pod_info{pod=~\"$worker\"},node)", + "definition": "label_values(kube_pod_info{cluster=~\"$cluster\", pod=~\"$worker\"},node)", "hide": 2, "includeAll": true, + "allValue": ".*", "label": "instance", "multi": true, "name": "hostname", "options": [], "query": { "qryType": 1, - "query": "label_values(kube_pod_info{pod=~\"$worker\"},node)", + "query": "label_values(kube_pod_info{cluster=~\"$cluster\", pod=~\"$worker\"},node)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, @@ -4363,16 +4405,17 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(node_uname_info{container=\"node-exporter\", nodename=~\"$hostname\"},instance)", + "definition": "label_values(node_uname_info{cluster=~\"$cluster\", container=\"node-exporter\", nodename=~\"$hostname\"},instance)", "hide": 2, "includeAll": true, + "allValue": ".*", "label": "ip", "multi": true, "name": "ip", "options": [], "query": { "qryType": 1, - "query": "label_values(node_uname_info{container=\"node-exporter\", nodename=~\"$hostname\"},instance)", + "query": "label_values(node_uname_info{cluster=~\"$cluster\", container=\"node-exporter\", nodename=~\"$hostname\"},instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, diff --git a/helm/soperator-monitoring-dashboards/templates/_helpers.tpl b/helm/soperator-monitoring-dashboards/templates/_helpers.tpl index 9a1ef297c..56ad39a81 100644 --- a/helm/soperator-monitoring-dashboards/templates/_helpers.tpl +++ b/helm/soperator-monitoring-dashboards/templates/_helpers.tpl @@ -37,3 +37,90 @@ grafana_dashboard: "1" {{- define "soperator-monitoring-dashboards.cmName" -}} {{- printf "%s-%s" .namePrefix .dashboardName | trunc 63 | trimSuffix "-" -}} {{- end }} + +{{/* Require every panel query to filter on the portable cluster variable. */}} +{{- define "soperator-monitoring-dashboards.validateClusterFilteredExpressions" -}} +{{- $path := .path -}} +{{- $value := .value -}} +{{- if kindIs "map" $value -}} + {{- range $key, $nested := $value -}} + {{- $nestedPath := printf "%s.%s" $path $key -}} + {{- if eq $key "expr" -}} + {{- if not (contains "cluster=~\"$cluster\"" (toString $nested)) -}} + {{- fail (printf "%s: %s must filter metrics with cluster=~\"$cluster\"" $path $nestedPath) -}} + {{- end -}} + {{- else -}} + {{- include "soperator-monitoring-dashboards.validateClusterFilteredExpressions" (dict "path" $nestedPath "value" $nested) -}} + {{- end -}} + {{- end -}} +{{- else if kindIs "slice" $value -}} + {{- range $index, $nested := $value -}} + {{- include "soperator-monitoring-dashboards.validateClusterFilteredExpressions" (dict "path" (printf "%s[%d]" $path $index) "value" $nested) -}} + {{- end -}} +{{- end -}} +{{- end }} + +{{/* Fail chart rendering when a dashboard violates the basic O11y portability contract. */}} +{{- define "soperator-monitoring-dashboards.validateDashboardPortability" -}} +{{- $path := .path -}} +{{- $dashboard := .dashboard -}} +{{- $variables := dig "templating" "list" (list) $dashboard -}} +{{- if lt (len $variables) 3 -}} + {{- fail (printf "%s: expected $datasource, $o11y, and $cluster variables" $path) -}} +{{- end -}} +{{- $datasource := index $variables 0 -}} +{{- $o11y := index $variables 1 -}} +{{- $cluster := index $variables 2 -}} + +{{- if ne (dig "name" "" $datasource) "datasource" -}} + {{- fail (printf "%s: first variable must be $datasource" $path) -}} +{{- end -}} +{{- if or (ne (dig "type" "" $datasource) "datasource") (ne (toString (dig "hide" 0 $datasource)) "2") -}} + {{- fail (printf "%s: $datasource must be a hidden datasource variable" $path) -}} +{{- end -}} +{{- if ne (dig "query" "" $datasource) "prometheus" -}} + {{- fail (printf "%s: $datasource must select prometheus datasources" $path) -}} +{{- end -}} + +{{- if ne (dig "name" "" $o11y) "o11y" -}} + {{- fail (printf "%s: second variable must be $o11y" $path) -}} +{{- end -}} +{{- if or (ne (dig "type" "" $o11y) "adhoc") (ne (toString (dig "hide" 0 $o11y)) "2") -}} + {{- fail (printf "%s: $o11y must be a hidden adhoc variable" $path) -}} +{{- end -}} +{{- if or (ne (len (dig "filters" (list) $o11y)) 0) (ne (len (dig "baseFilters" (list) $o11y)) 0) -}} + {{- fail (printf "%s: $o11y must have no in-cluster filters" $path) -}} +{{- end -}} +{{- if ne (dig "datasource" "uid" "" $o11y) "${datasource}" -}} + {{- fail (printf "%s: $o11y must use ${datasource}" $path) -}} +{{- end -}} + +{{- if ne (dig "name" "" $cluster) "cluster" -}} + {{- fail (printf "%s: third variable must be $cluster" $path) -}} +{{- end -}} +{{- if or (ne (dig "type" "" $cluster) "query") (ne (toString (dig "hide" 0 $cluster)) "2") -}} + {{- fail (printf "%s: $cluster must be a hidden query variable" $path) -}} +{{- end -}} +{{- if or (not (dig "includeAll" false $cluster)) (ne (dig "allValue" "" $cluster) ".*") -}} + {{- fail (printf "%s: $cluster must include All with the value .*" $path) -}} +{{- end -}} +{{- if ne (dig "current" "value" "" $cluster) "$__all" -}} + {{- fail (printf "%s: $cluster must default to All" $path) -}} +{{- end -}} +{{- if ne (dig "datasource" "uid" "" $cluster) "${datasource}" -}} + {{- fail (printf "%s: $cluster must use ${datasource}" $path) -}} +{{- end -}} + +{{- range $variable := $variables -}} + {{- if and (eq (dig "type" "" $variable) "query") (ne (dig "name" "" $variable) "cluster") -}} + {{- $name := dig "name" "" $variable -}} + {{- if not (contains "cluster=~\"$cluster\"" (dig "definition" "" $variable)) -}} + {{- fail (printf "%s: $%s definition must filter metrics with cluster=~\"$cluster\"" $path $name) -}} + {{- end -}} + {{- if not (contains "cluster=~\"$cluster\"" (dig "query" "query" "" $variable)) -}} + {{- fail (printf "%s: $%s query must filter metrics with cluster=~\"$cluster\"" $path $name) -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- include "soperator-monitoring-dashboards.validateClusterFilteredExpressions" (dict "path" $path "value" $dashboard) -}} +{{- end -}} diff --git a/helm/soperator-monitoring-dashboards/templates/configmaps.yaml b/helm/soperator-monitoring-dashboards/templates/configmaps.yaml index ab8560e00..0a4e8379b 100644 --- a/helm/soperator-monitoring-dashboards/templates/configmaps.yaml +++ b/helm/soperator-monitoring-dashboards/templates/configmaps.yaml @@ -1,7 +1,10 @@ {{- $namePrefix := include "soperator-monitoring-dashboards.namePrefix" . -}} {{- range $path, $file := .Files.Glob "dashboards/*.json" }} +{{- $dashboard := mustFromJson (toString $file) -}} +{{- include "soperator-monitoring-dashboards.validateDashboardPortability" (dict "path" $path "dashboard" $dashboard) -}} {{- $dashboardName := include "soperator-monitoring-dashboards.dashboardName" $path -}} {{- $cmName := include "soperator-monitoring-dashboards.cmName" (dict "namePrefix" $namePrefix "dashboardName" $dashboardName) -}} +--- apiVersion: v1 kind: ConfigMap metadata: @@ -18,5 +21,4 @@ metadata: data: {{ printf "%s.json" $dashboardName }}: | {{- toString $file | nindent 4 }} ---- -{{- end }} +{{ end }} diff --git a/internal/exporter/collector.go b/internal/exporter/collector.go index 0d13ffc0a..0e55472c2 100644 --- a/internal/exporter/collector.go +++ b/internal/exporter/collector.go @@ -96,6 +96,7 @@ func newMetricsCollector( var nodeInfoLabels = []string{ "node_name", "instance_id", + "nodeset_name", "state_base", "state_is_drain", "state_is_maintenance", @@ -634,6 +635,7 @@ func (c *MetricsCollector) slurmNodeMetrics( labels := []string{ node.Name, node.InstanceID, + topology.SlurmNodeSetName, string(node.BaseState()), strconv.FormatBool(node.IsDrainState()), // Keep "true"/"false" for backward compatibility strconv.FormatBool(node.IsMaintenanceState()), // Keep "true"/"false" for backward compatibility diff --git a/internal/exporter/collector_test.go b/internal/exporter/collector_test.go index c3ad73d8a..41ef8c4d5 100644 --- a/internal/exporter/collector_test.go +++ b/internal/exporter/collector_test.go @@ -213,7 +213,7 @@ func TestMetricsCollector_Describe(t *testing.T) { } // Base metrics - assert.Contains(t, found, `Desc{fqName: "slurm_node_info", help: "Slurm node info", constLabels: {}, variableLabels: {node_name,instance_id,state_base,state_is_drain,state_is_maintenance,state_is_reserved,state_is_completing,state_is_fail,state_is_planned,state_is_not_responding,state_is_invalid,state_is_cloud,state_is_power_down,state_is_power_drain,state_is_powered_down,state_is_powering_down,state_is_powering_up,state_is_power_up,is_unavailable,reservation_name,address,reason,comment}}`) + assert.Contains(t, found, `Desc{fqName: "slurm_node_info", help: "Slurm node info", constLabels: {}, variableLabels: {node_name,instance_id,nodeset_name,state_base,state_is_drain,state_is_maintenance,state_is_reserved,state_is_completing,state_is_fail,state_is_planned,state_is_not_responding,state_is_invalid,state_is_cloud,state_is_power_down,state_is_power_drain,state_is_powered_down,state_is_powering_down,state_is_powering_up,state_is_power_up,is_unavailable,reservation_name,address,reason,comment}}`) assert.Contains(t, found, `Desc{fqName: "slurm_node_cpus_total", help: "Total CPUs on the node", constLabels: {}, variableLabels: {node_name}}`) assert.Contains(t, found, `Desc{fqName: "slurm_node_cpus_allocated", help: "CPUs allocated on the node", constLabels: {}, variableLabels: {node_name}}`) assert.Contains(t, found, `Desc{fqName: "slurm_node_cpus_idle", help: "Idle CPUs on the node", constLabels: {}, variableLabels: {node_name}}`) @@ -253,7 +253,10 @@ func TestMetricsCollector_NodeTopologyMetrics(t *testing.T) { }) collector := newMetricsCollector(mockClient, slurmapi.ListJobsParams{}, source) - mockClient.EXPECT().ListNodes(mock.Anything).Return([]slurmapi.Node{testNode("worker-0")}, nil).Once() + mockClient.EXPECT().ListNodes(mock.Anything).Return([]slurmapi.Node{ + testNode("worker-0"), + testNode("worker-without-topology"), + }, nil).Once() mockClient.EXPECT().ListJobsWithParams(mock.Anything, mock.Anything).Return([]slurmapi.Job{ {ID: 123, State: "RUNNING", Nodes: "worker-0"}, }, nil).Once() @@ -275,6 +278,8 @@ func TestMetricsCollector_NodeTopologyMetrics(t *testing.T) { assert.Contains(t, metricsText, `GAUGE; slurm_node_nvlink_instance_group{instance_id="worker-0-instance",node_name="worker-0",nodeset_name="gpu-workers",nvlink_instance_group="nvlig-1"} 1`) assert.Contains(t, metricsText, `GAUGE; slurm_node_job{job_id="123",node_name="worker-0",nodeset_name="gpu-workers",nvlink_instance_group="nvlig-1"} 1`) + assertMetricHasLabels(t, metricsText, []string{`node_name="worker-0"`, `nodeset_name="gpu-workers"`}) + assertMetricHasLabels(t, metricsText, []string{`node_name="worker-without-topology"`, `nodeset_name=""`}) } func TestMetricsCollector_NodeTopologyRefreshRetriesAfterTimeout(t *testing.T) { diff --git a/internal/exporter/kubernetes_topology.go b/internal/exporter/kubernetes_topology.go index b6d6830ea..cf79ad416 100644 --- a/internal/exporter/kubernetes_topology.go +++ b/internal/exporter/kubernetes_topology.go @@ -23,11 +23,11 @@ type kubernetesNodeTopologySource struct { } func topologyNodeSelector() (labels.Selector, error) { - nodeSetRequirement, err := labels.NewRequirement(slurmNodeSetNameLabel, selection.Exists, nil) + nvlinkRequirement, err := labels.NewRequirement(nvlinkInstanceGroupLabel, selection.Exists, nil) if err != nil { - return nil, fmt.Errorf("create Slurm NodeSet label selector: %w", err) + return nil, fmt.Errorf("create NVLink instance group label selector: %w", err) } - return labels.NewSelector().Add(*nodeSetRequirement), nil + return labels.NewSelector().Add(*nvlinkRequirement), nil } // NewKubernetesNodeTopologyCache creates a list/watch cache scoped to the Kubernetes objects @@ -110,6 +110,10 @@ func (s *kubernetesNodeTopologySource) ListNodeTopologies(ctx context.Context) ( podsByKubernetesNode := make(map[string][]string, len(pods.Items)) for i := range pods.Items { pod := &pods.Items[i] + topologies[pod.Name] = NodeTopology{ + KubernetesNode: pod.Spec.NodeName, + SlurmNodeSetName: pod.Labels[consts.LabelNodeSetKey], + } if pod.Spec.NodeName == "" { continue } @@ -125,12 +129,9 @@ func (s *kubernetesNodeTopologySource) ListNodeTopologies(ctx context.Context) ( return nil, fmt.Errorf("get Kubernetes node %q: %w", kubernetesNode, err) } - topology := NodeTopology{ - KubernetesNode: kubernetesNode, - NVLinkInstanceGroup: node.Labels[nvlinkInstanceGroupLabel], - SlurmNodeSetName: node.Labels[slurmNodeSetNameLabel], - } for _, podName := range podNames { + topology := topologies[podName] + topology.NVLinkInstanceGroup = node.Labels[nvlinkInstanceGroupLabel] topologies[podName] = topology } } @@ -151,6 +152,7 @@ func transformWorkerPodForTopology(obj any) (any, error) { ResourceVersion: pod.ResourceVersion, Labels: map[string]string{ consts.LabelInstanceKey: pod.Labels[consts.LabelInstanceKey], + consts.LabelNodeSetKey: pod.Labels[consts.LabelNodeSetKey], consts.LabelWorkerKey: pod.Labels[consts.LabelWorkerKey], }, } @@ -172,7 +174,6 @@ func transformNodeForTopology(obj any) (any, error) { ResourceVersion: node.ResourceVersion, Labels: map[string]string{ nvlinkInstanceGroupLabel: node.Labels[nvlinkInstanceGroupLabel], - slurmNodeSetNameLabel: node.Labels[slurmNodeSetNameLabel], }, } node.Spec = corev1.NodeSpec{} diff --git a/internal/exporter/kubernetes_topology_test.go b/internal/exporter/kubernetes_topology_test.go index dec18ffde..ac656459b 100644 --- a/internal/exporter/kubernetes_topology_test.go +++ b/internal/exporter/kubernetes_topology_test.go @@ -50,17 +50,30 @@ func TestKubernetesNodeTopologySource(t *testing.T) { Namespace: "slurm", Labels: map[string]string{ consts.LabelInstanceKey: "cluster-a", + consts.LabelNodeSetKey: "gpu-workers", consts.LabelWorkerKey: consts.LabelWorkerValue, }, }, Spec: corev1.PodSpec{NodeName: "k8s-node-1"}, }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "worker-1", + Namespace: "slurm", + Labels: map[string]string{ + consts.LabelInstanceKey: "cluster-a", + consts.LabelNodeSetKey: "gpu-workers", + consts.LabelWorkerKey: consts.LabelWorkerValue, + }, + }, + }, &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "other-worker-0", Namespace: "slurm", Labels: map[string]string{ consts.LabelInstanceKey: "cluster-b", + consts.LabelNodeSetKey: "other-workers", consts.LabelWorkerKey: consts.LabelWorkerValue, }, }, @@ -70,8 +83,8 @@ func TestKubernetesNodeTopologySource(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "k8s-node-1", Labels: map[string]string{ - nvlinkInstanceGroupLabel: "nvlig-1", - slurmNodeSetNameLabel: "gpu-workers", + nvlinkInstanceGroupLabel: "nvlig-1", + "slurm.nebius.ai/nodeset-name": "infrastructure-node-pool", }, }, }, @@ -87,6 +100,9 @@ func TestKubernetesNodeTopologySource(t *testing.T) { NVLinkInstanceGroup: "nvlig-1", SlurmNodeSetName: "gpu-workers", }, + "worker-1": { + SlurmNodeSetName: "gpu-workers", + }, }, topologies) } @@ -94,8 +110,8 @@ func TestTopologyNodeSelector(t *testing.T) { selector, err := topologyNodeSelector() require.NoError(t, err) - assert.True(t, selector.Matches(labels.Set{slurmNodeSetNameLabel: "gpu-workers"})) - assert.True(t, selector.Matches(labels.Set{slurmNodeSetNameLabel: ""})) + assert.True(t, selector.Matches(labels.Set{nvlinkInstanceGroupLabel: "nvlig-1"})) + assert.True(t, selector.Matches(labels.Set{nvlinkInstanceGroupLabel: ""})) assert.False(t, selector.Matches(labels.Set{})) } @@ -118,6 +134,7 @@ func TestKubernetesNodeTopologySourceGetsEachUsedNodeOnce(t *testing.T) { Namespace: "slurm", Labels: map[string]string{ consts.LabelInstanceKey: "cluster-a", + consts.LabelNodeSetKey: "gpu-workers", consts.LabelWorkerKey: consts.LabelWorkerValue, }, }, @@ -129,6 +146,7 @@ func TestKubernetesNodeTopologySourceGetsEachUsedNodeOnce(t *testing.T) { Namespace: "slurm", Labels: map[string]string{ consts.LabelInstanceKey: "cluster-a", + consts.LabelNodeSetKey: "gpu-workers", consts.LabelWorkerKey: consts.LabelWorkerValue, }, }, @@ -139,7 +157,6 @@ func TestKubernetesNodeTopologySourceGetsEachUsedNodeOnce(t *testing.T) { Name: "k8s-node-1", Labels: map[string]string{ nvlinkInstanceGroupLabel: "nvlig-1", - slurmNodeSetNameLabel: "gpu-workers", }, }, }, @@ -190,6 +207,7 @@ func TestTransformWorkerPodForTopology(t *testing.T) { ResourceVersion: "123", Labels: map[string]string{ consts.LabelInstanceKey: "cluster-a", + consts.LabelNodeSetKey: "gpu-workers", consts.LabelWorkerKey: consts.LabelWorkerValue, "irrelevant": "value", }, @@ -211,6 +229,7 @@ func TestTransformWorkerPodForTopology(t *testing.T) { assert.Equal(t, "123", actual.ResourceVersion) assert.Equal(t, map[string]string{ consts.LabelInstanceKey: "cluster-a", + consts.LabelNodeSetKey: "gpu-workers", consts.LabelWorkerKey: consts.LabelWorkerValue, }, actual.Labels) assert.Equal(t, corev1.PodSpec{NodeName: "k8s-node-1"}, actual.Spec) @@ -225,7 +244,6 @@ func TestTransformNodeForTopology(t *testing.T) { ResourceVersion: "456", Labels: map[string]string{ nvlinkInstanceGroupLabel: "nvlig-1", - slurmNodeSetNameLabel: "gpu-workers", "irrelevant": "value", }, Annotations: map[string]string{"irrelevant": "value"}, @@ -242,7 +260,6 @@ func TestTransformNodeForTopology(t *testing.T) { assert.Equal(t, "456", actual.ResourceVersion) assert.Equal(t, map[string]string{ nvlinkInstanceGroupLabel: "nvlig-1", - slurmNodeSetNameLabel: "gpu-workers", }, actual.Labels) assert.Empty(t, actual.Spec) assert.Empty(t, actual.Status) diff --git a/internal/exporter/topology.go b/internal/exporter/topology.go index 567b4bb4f..1e552b498 100644 --- a/internal/exporter/topology.go +++ b/internal/exporter/topology.go @@ -2,19 +2,16 @@ package exporter import "context" -const ( - nvlinkInstanceGroupLabel = "topology.nebius.com/nvl-instance-group-id" - slurmNodeSetNameLabel = "slurm.nebius.ai/nodeset-name" -) +const nvlinkInstanceGroupLabel = "topology.nebius.com/nvl-instance-group-id" -// NodeTopology describes the Kubernetes placement metadata for a Slurm node. +// NodeTopology describes Soperator NodeSet membership and Kubernetes placement metadata for a Slurm node. type NodeTopology struct { KubernetesNode string NVLinkInstanceGroup string SlurmNodeSetName string } -// NodeTopologySource resolves Slurm worker names to Kubernetes node topology. +// NodeTopologySource resolves Slurm worker names to Soperator NodeSet and Kubernetes node topology. type NodeTopologySource interface { ListNodeTopologies(ctx context.Context) (map[string]NodeTopology, error) }