Watch DCP fields per GPU model to fix startup on mixed-model nodes

internal/pkg/collector/gpu_collector.go

@@ -260,13 +260,40 @@ func (c *DCGMCollector) latestValues(mi devicemonitoring.Info) ([]dcgm.FieldValu
 		)
 	}
 
+	// Watch registration is scoped per GPU model (see the devicewatcher
+	// package), but the field list above isn't - it's every counter
+	// configured for this entity type, regardless of which model actually
+	// had it watched. For an ordinary field that's fine: DCGM reports one
+	// it doesn't support as a per-entity blank value. For a DCP field it
+	// isn't: asking for a value that was never watched returns
+	// DCGM_ST_NOT_WATCHED, and the stale-watch repair path then retries
+	// forever trying to fix a watch that was deliberately never created.
+	// Filter to what this entity's model actually has watched.
+	fields = filterDCPFieldsForModel(fields, mi.DeviceInfo)
+
 	return dcgmprovider.Client().EntityGetLatestValues(
 		mi.Entity.EntityGroupId,
 		mi.Entity.EntityId,
 		fields,
 	)
 }
 
+// filterDCPFieldsForModel drops DCP fields the entity's GPU model doesn't
+// support, leaving non-DCP fields untouched. See NVIDIA/dcgm-exporter#736.
+func filterDCPFieldsForModel(fields []dcgm.Short, device dcgm.Device) []dcgm.Short {
+	if device.Identifiers.Model == "" {
+		return fields
+	}
+
+	filtered := make([]dcgm.Short, 0, len(fields))
+	for _, fieldID := range fields {
+		if devicewatcher.ModelSupportsDCPField(device.Identifiers.Model, device.GPU, fieldID) {
+			filtered = append(filtered, fieldID)
+		}
+	}
+	return filtered
+}
+
 // addMetrics renders values with the labels and identity fields for their entity type.
 func (c *DCGMCollector) addMetrics(
 	metrics MetricsByCounter,

internal/pkg/counters/const.go

@@ -27,3 +27,12 @@ const (
 	DCGMExpXIDErrorsTotal   = "DCGM_EXP_XID_ERRORS_TOTAL"
 	DCGMExpClockEventsTotal = "DCGM_EXP_CLOCK_EVENTS_TOTAL"
 )
+
+// IsDCPField reports whether fieldID falls in the DCP/profiling field ID range
+// (DCGM_FI_PROF_*). These fields are validated for the whole DCGM watch group
+// at registration time, unlike ordinary fields which degrade to a per-entity
+// NOT_SUPPORTED value at scrape time - see devicewatcher's model-partitioned
+// watch path for why that distinction matters.
+func IsDCPField(fieldID uint) bool {
+	return fieldID >= dcpFieldsStart && fieldID < cpuFieldsStart
+}

internal/pkg/counters/counter_config.go

@@ -191,7 +191,7 @@ func ExtractCounters(records [][]string, c *appconfig.Config) (*CounterSet, erro
 }
 
 func fieldIsSupported(fieldID uint, c *appconfig.Config) bool {
-	if fieldID < dcpFieldsStart || fieldID >= cpuFieldsStart {
+	if !IsDCPField(fieldID) {
 		return true
 	}