Little fixes to the prometheus sink (#115)

* Add uint64 to float64 cast option * Add prometheus sink to the list of available sinks * Add aggregated counters by gpu for nvlink errors --------- Co-authored-by: Michael Schwarz <[email protected]>
ClusterCockpit · Nov 22, 2024 · 26ce177 · 26ce177
1 parent 8837ff4
commit 26ce177
Show file tree

Hide file tree

Showing 3 changed files with 56 additions and 0 deletions.
diff --git a/collectors/nvidiaMetric.go b/collectors/nvidiaMetric.go
@@ -941,13 +941,20 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro
 	//
 	// For Pascal &tm; or newer fully supported devices.
 
+	var aggregate_crc_errors uint64 = 0
+	var aggregate_ecc_errors uint64 = 0
+	var aggregate_replay_errors uint64 = 0
+	var aggregate_recovery_errors uint64 = 0
+	var aggregate_crc_flit_errors uint64 = 0
+
 	for i := 0; i < nvml.NVLINK_MAX_LINKS; i++ {
 		state, ret := nvml.DeviceGetNvLinkState(device.device, i)
 		if ret == nvml.SUCCESS {
 			if state == nvml.FEATURE_ENABLED {
 				if !device.excludeMetrics["nv_nvlink_crc_errors"] {
 					// Data link receive data CRC error counter
 					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_CRC_DATA)
+					aggregate_crc_errors = aggregate_crc_errors + count
 					if ret == nvml.SUCCESS {
 						y, err := lp.New("nv_nvlink_crc_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
 						if err == nil {
@@ -960,6 +967,7 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro
 				if !device.excludeMetrics["nv_nvlink_ecc_errors"] {
 					// Data link receive data ECC error counter
 					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_ECC_DATA)
+					aggregate_ecc_errors = aggregate_ecc_errors + count
 					if ret == nvml.SUCCESS {
 						y, err := lp.New("nv_nvlink_ecc_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
 						if err == nil {
@@ -972,6 +980,7 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro
 				if !device.excludeMetrics["nv_nvlink_replay_errors"] {
 					// Data link transmit replay error counter
 					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_REPLAY)
+					aggregate_replay_errors = aggregate_replay_errors + count
 					if ret == nvml.SUCCESS {
 						y, err := lp.New("nv_nvlink_replay_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
 						if err == nil {
@@ -984,6 +993,7 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro
 				if !device.excludeMetrics["nv_nvlink_recovery_errors"] {
 					// Data link transmit recovery error counter
 					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_RECOVERY)
+					aggregate_recovery_errors = aggregate_recovery_errors + count
 					if ret == nvml.SUCCESS {
 						y, err := lp.New("nv_nvlink_recovery_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
 						if err == nil {
@@ -996,6 +1006,7 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro
 				if !device.excludeMetrics["nv_nvlink_crc_flit_errors"] {
 					// Data link receive flow control digit CRC error counter
 					count, ret := nvml.DeviceGetNvLinkErrorCounter(device.device, i, nvml.NVLINK_ERROR_DL_CRC_FLIT)
+					aggregate_crc_flit_errors = aggregate_crc_flit_errors + count
 					if ret == nvml.SUCCESS {
 						y, err := lp.New("nv_nvlink_crc_flit_errors", device.tags, device.meta, map[string]interface{}{"value": count}, time.Now())
 						if err == nil {
@@ -1008,6 +1019,48 @@ func readNVLinkStats(device NvidiaCollectorDevice, output chan lp.CCMetric) erro
 			}
 		}
 	}
+
+	// Export aggegated values
+	if !device.excludeMetrics["nv_nvlink_crc_errors"] {
+		// Data link receive data CRC error counter
+		y, err := lp.New("nv_nvlink_crc_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_crc_errors}, time.Now())
+		if err == nil {
+			y.AddTag("stype", "nvlink")
+			output <- y
+		}
+	}
+	if !device.excludeMetrics["nv_nvlink_ecc_errors"] {
+		// Data link receive data ECC error counter
+		y, err := lp.New("nv_nvlink_ecc_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_ecc_errors}, time.Now())
+		if err == nil {
+			y.AddTag("stype", "nvlink")
+			output <- y
+		}
+	}
+	if !device.excludeMetrics["nv_nvlink_replay_errors"] {
+		// Data link transmit replay error counter
+		y, err := lp.New("nv_nvlink_replay_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_replay_errors}, time.Now())
+		if err == nil {
+			y.AddTag("stype", "nvlink")
+			output <- y
+		}
+	}
+	if !device.excludeMetrics["nv_nvlink_recovery_errors"] {
+		// Data link transmit recovery error counter
+		y, err := lp.New("nv_nvlink_recovery_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_recovery_errors}, time.Now())
+		if err == nil {
+			y.AddTag("stype", "nvlink")
+			output <- y
+		}
+	}
+	if !device.excludeMetrics["nv_nvlink_crc_flit_errors"] {
+		// Data link receive flow control digit CRC error counter
+		y, err := lp.New("nv_nvlink_crc_flit_errors_sum", device.tags, device.meta, map[string]interface{}{"value": aggregate_crc_flit_errors}, time.Now())
+		if err == nil {
+			y.AddTag("stype", "nvlink")
+			output <- y
+		}
+	}
 	return nil
 }
 

diff --git a/sinks/prometheusSink.go b/sinks/prometheusSink.go
@@ -49,6 +49,8 @@ func intToFloat64(input interface{}) (float64, error) {
 		return float64(value), nil
 	case int64:
 		return float64(value), nil
+	case uint64:
+		return float64(value), nil
 	}
 	return 0, errors.New("cannot cast value to float64")
 }

diff --git a/sinks/sinkManager.go b/sinks/sinkManager.go
@@ -21,6 +21,7 @@ var AvailableSinks = map[string]func(name string, config json.RawMessage) (Sink,
 	"influxdb":    NewInfluxSink,
 	"influxasync": NewInfluxAsyncSink,
 	"http":        NewHttpSink,
+	"prometheus":  NewPrometheusSink,
 }
 
 // Metric collector manager data structure