Added DCGM metrics for those with Quadro cards.

Updated include paths for certain CUDA installations

Makefile

@@ -2,14 +2,20 @@
 
 OBJS = evga-icx.o evga-card.o icx3.o board-sensors.o zen3-rapl.o
 LDLIBS = -li2c -lm
-CFLAGS = -MD
+CFLAGS = -MD -I/usr/local/cuda/targets/x86_64-linux/include/
 
 ifdef USE_NVML
-LDLIBS += -lnvidia-ml
+LDLIBS += -lnvidia-ml -L/usr/local/cuda/targets/x86_64-linux/lib/stubs/
 CFLAGS += -DUSE_NVML
 OBJS += nvidia-sensors.o
 endif
 
+ifdef USE_DCGM
+LDLIBS += -ldcgm
+CFLAGS += -DUSE_DCGM -I/usr/include/datacenter-gpu-manager-4/
+OBJS += nvidia-dcgm.o
+endif
+
 ifdef USE_LIBPCI
 LDLIBS += -lpci
 CFLAGS += -DUSE_LIBPCI

README.md

@@ -36,6 +36,9 @@ Add the make flag `USE_NVML=1` and the it will also display the main GPU tempera
 ### VRAM and Hotspot temperature
 Add the make flag `USE_LIBPCI=1` and you can also read the VRAM and "hotspot" temperatures.  These require direct memory access to the PCI device so you must run as root and also enable the kernel parameter `iomem=relaxed`.  These sensors are **extremely** undocumented so I can't say anything about their accuracy.
 
+### Hardware monitoring
+This program can also monitor hardware sensors using the standard linux `hwmon` API.  See board-sensors.h for examples of how to configure these, they must be staticlly compiled in.
+
 ## Usage
 Note that when controlling fans directly through iCX3 they will fall offline from the Nvidia driver and show as 0 RPM until you return them to automatic mode.
 

board-sensors.h

@@ -20,28 +20,28 @@ static struct hwmon_sensor hwmon_sensor_info[] =
 {
 	{"zenpower",	"temp1",	"CPU ",			"CPU",	"°C",	1000.0,	-40.0	},	/* Tdie */
 	{"asusec",		"temp2",	"Motherboard ",	"CPU",	"°C",	1000.0,	-40.0	},	/* CPU */
-	{"zenpower",	"temp3",	"CPU ",			"CCX",	"°C",	1000.0,	-40.0	},	/* Tccd1 */
+	{"zenpower",	"temp3",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd1 */
-	{"zenpower",	"temp4",	"CPU ",			"CCX",	"°C",	1000.0,	-40.0	},	/* Tccd2 */
+	{"zenpower",	"temp4",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd2 */
-	{"zenpower",	"temp5",	"CPU ",			"CCX",	"°C",	1000.0,	-40.0	},	/* Tccd3 */
+	{"zenpower",	"temp5",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd3 */
-	{"zenpower",	"temp6",	"CPU ",			"CCX",	"°C",	1000.0,	-40.0	},	/* Tccd4 */
+	{"zenpower",	"temp6",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd4 */
-	{"zenpower",	"temp7",	"CPU ",			"CCX",	"°C",	1000.0,	-40.0	},	/* Tccd5 */
+	{"zenpower",	"temp7",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd5 */
-	{"zenpower",	"temp8",	"CPU ",			"CCX",	"°C",	1000.0,	-40.0	},	/* Tccd6 */
+	{"zenpower",	"temp8",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd6 */
-	{"zenpower",	"temp9",	"CPU ",			"CCX",	"°C",	1000.0,	-40.0	},	/* Tccd7 */
+	{"zenpower",	"temp9",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd7 */
-	{"zenpower",	"temp10",	"CPU ",			"CCX",	"°C",	1000.0,	-40.0	},	/* Tccd8 */
+	{"zenpower",	"temp10",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd8 */
 	{"zen-rapl",	"",			"CPU ",			"POW",	" W",	0.0,	0.0		},	/* Zen RAPL placeholder */
 	{"asusec",		"temp1",	"Motherboard ",	"CHIP",	"°C",	1000.0,	-40.0	},	/* Chipset */
 	{"asusec",		"temp5",	"Motherboard ",	"VRM",	"°C",	1000.0,	-40.0	},	/* VRM */
 	{"asusec",		"temp3",	"",				"MOBO",	"°C",	1000.0,	-40.0	},	/* Motherboard */
-	{"asusec",		"temp4",	"Motherboard ",	"SENS",	"°C",	1000.0,	-40.0	},	/* T_Sensor */
-	{"asusec",		"temp6",	"Motherboard ",	"H2O",	"°C",	1000.0,	-40.0	},	/* Water_In */
-	{"asusec",		"temp7",	"Motherboard ",	"H2O",	"°C",	1000.0,	-40.0	},	/* Water_Out */
 	{"nvme",		"temp1",	"NVMe ",		"NVME",	"°C",	1000.0,	-40.0	},	/* NVME Composite */
-	{"nct6798",		"fan2",		"CPU fan",		"CPU",	"%",	15.0,	0.0		},	/* cpu_fan */
+	{"asusec",		"temp6",	"Motherboard ",	"AIR",	"°C",	1000.0,	-40.0	},	/* Water_In */
-	{"asusec",		"fan1",		"",				"CPU",	"%",	15.0,	0.0		},	/* cpu_opt */
+	{"asusec",		"temp4",	"Motherboard ",	"AIR",	"°C",	1000.0,	-40.0	},	/* T_Sensor */
-	{"nct6798",		"fan1",		"Chassis 1",	"CHA",	"%",	12.0,	0.0		},	/* cha1? */
+	{"asusec",		"temp7",	"Motherboard ",	"AIR",	"°C",	1000.0,	-40.0	},	/* Water_Out */
-	{"nct6798",		"fan3",		"Chassis 2",	"CHA",	"%",	12.0,	0.0		},	/* cha2? */
+	{"nct6798",		"fan2",		"CPU fan",		"CPU",	"%",	15.0,	0.0		},	/* cpu_fan, cpu mid */
-	{"nct6798",		"fan4",		"Chassis 3",	"CHA",	"%",	13.0,	0.0		},	/* cha3 */
+	{"asusec",		"fan1",		"",				"CPU",	"%",	15.0,	0.0		},	/* cpu_opt, cpu front */
-	{"nct6798",		"fan5",		"H amp",		"CHA",	"%",	12.0,	0.0		},	/* h_amp? */
+	{"nct6798",		"fan5",		"H amp",		"CHA",	"%",	12.0,	0.0		},	/* h_amp, front fan */
+	{"nct6798",		"fan3",		"Chassis 2",	"CHA",	"%",	12.0,	0.0		},	/* cha2, top front */
+	{"nct6798",		"fan1",		"Chassis 1",	"CHA",	"%",	12.0,	0.0		},	/* cha1, top rear */
+	{"nct6798",		"fan4",		"Chassis 3",	"CHA",	"%",	13.0,	0.0		},	/* cha3, rear */
 	{"nct6798",		"fan6",		"AIO pump",		"CHA",	"%",	12.0,	0.0		},	/* aio_pump? */
 	{"nct6798",		"fan7",		"W pump",		"CHA",	"%",	12.0,	0.0		},	/* w_pump+? */
 	{"asusec",		"fan2",		"",				"CHIP",	"%",	35.0,	0.0		},	/* chipset */

evga-card.c

@@ -10,9 +10,10 @@
    and respond with the correct iCX3 version information */
 int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus, int i2c_bus)
 {
-	char i2c_devices_path[NAME_MAX];
+	char i2c_devices_path[PATH_MAX];
-	char device_path[NAME_MAX];
+	char device_path[PATH_MAX];
-	char dev_file[NAME_MAX];
+	char dev_file[PATH_MAX];
+	char pci_path[PATH_MAX];
 	char *pci_addr;
 
 	FILE *test_fd;
@@ -55,10 +56,14 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus, int i2c_bus)
 			continue;
 		
 		/* Read the PCI info for the underlying device */
-		pci_vendor           = read_pci_id(device_path, "/device/vendor");
+		strcat(device_path, "/../");
-		pci_device           = read_pci_id(device_path, "/device/device");
+		if (realpath(device_path, pci_path) == NULL)
-		pci_subsystem_vendor = read_pci_id(device_path, "/device/subsystem_vendor");
+			continue;
-		pci_subsystem_device = read_pci_id(device_path, "/device/subsystem_device");
+		
+		pci_vendor           = read_pci_id(pci_path, "/vendor");
+		pci_device           = read_pci_id(pci_path, "/device");
+		pci_subsystem_vendor = read_pci_id(pci_path, "/subsystem_vendor");
+		pci_subsystem_device = read_pci_id(pci_path, "/subsystem_device");
 		
 		/* See if it's a matching device for a supported EVGA card */
 		for (int i = 0; i < (sizeof(evga_pci_ids) / sizeof(struct gpu_pci_info)); i++) {
@@ -97,7 +102,7 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus, int i2c_bus)
 unsigned short read_pci_id(char *device_path, char *field)
 {
 	char buf[16];
-	char file_path[NAME_MAX];
+	char file_path[PATH_MAX];
 	
 	strcpy(file_path, device_path);
 	strcat(file_path, field);
@@ -119,7 +124,7 @@ unsigned short read_pci_id(char *device_path, char *field)
 
 char *read_nvidia_pci_address(char *device_path)
 {
-	char file_path[NAME_MAX];
+	char file_path[PATH_MAX];
 	
 	char *ret = calloc(16 + 1, sizeof(char)); /* assuming pci ids could look as large as 00000000:0C:00.0 */
 	

evga-icx.c

@@ -6,11 +6,15 @@
 #include <stdlib.h>
 
 #ifdef USE_NVML
-#include "nvidia-sensors.h"
+	#include "nvidia-sensors.h"
+#endif
+
+#ifdef USE_DCGM
+	#include "nvidia-dcgm.h"
 #endif
 
 #ifdef USE_LIBPCI
-#include "gddr6.h"
+	#include "gddr6.h"
 #endif
 
 #include "icx3.h"
@@ -46,7 +50,7 @@ static const char helpstring[] = "Available options:\n"
 								 "--overwrite  : Overwrite previously displayed info with --watch and --compact instead of continuously logging\n"
 								 "--color      : Print headers in color in --compact mode for better readability\n"
 								 "--no-reasons : Do not query NVML for clock reasons (can cause stuttering)\n"
-								 "--board      : Also print temperatures from the CPU, motherboard, and other sensors";
+								 "--board      : Also print temperatures from the CPU, motherboard, and other sensors\n";
 								 
 void print_gpu_info(int gpu_num, struct card_info gpus[]);
 void print_board_info(struct hwmon_avail_sensor *board_sensors, int num_sensors);
@@ -153,7 +157,6 @@ int main (int argc, char **argv)
 		return -1;
 	} else if (gpu_count == 0) {
 		printf("No supported GPUs found.\nAre you root or do you have udev access to i2c devices?\nDo you need to run `modprobe i2c-dev`?\n");
-		return -1;
 	}
 	/* Check for invalid GPUs */
 	if (gpu_num > gpu_count - 1) {
@@ -195,6 +198,10 @@ int main (int argc, char **argv)
 	}
 #endif
 
+#ifdef USE_DCGM
+	init_dcgm();
+#endif
+
 	/* PCI init for VRAM/hotspot temps */
 #ifdef USE_LIBPCI
 	for (int i = 0; i < gpu_count; i++)
@@ -209,6 +216,10 @@ int main (int argc, char **argv)
 			
 			if (print_board_sensors)
 				print_board_info(board_sensors, board_sensor_count);
+			
+#ifdef USE_DCGM
+			print_dcgm(compact, overwrite);
+#endif
 
 			if (gpu_num == -1) {
 				/* No GPU specified on command line, loop over all supported GPUs */
@@ -238,6 +249,9 @@ int main (int argc, char **argv)
 #ifdef USE_NVML
 	nvmlShutdown();
 #endif
+#ifdef USE_DCGM
+	shutdown_dcgm();
+#endif
 }
 
 void print_board_info(struct hwmon_avail_sensor *board_sensors, int num_sensors)

icx3.h

@@ -1,6 +1,7 @@
 #ifndef ICX3_H
 #define ICX3_H
 
+#include <stdint.h>
 #include "evga-card.h"
 
 #define ICX3_I2C_ADDR 0x2D
@@ -22,26 +23,26 @@
 #define ICX3_WRITE_DISABLED 0xFE
 
 struct icx3_fan_control {
-	unsigned char length;
+	uint8_t length;
-	unsigned char fanmode;
+	uint8_t fanmode;
-	unsigned short rpm_offset;
+	uint16_t rpm_offset;
-	unsigned char duty;
+	uint8_t duty;
-	unsigned char duty_status;
+	uint8_t duty_status;
-	unsigned short rpm_status;
+	uint16_t rpm_status;
 };
 
 struct icx3_temp_sensors {
-	unsigned char length;
+	uint8_t length;
-	unsigned char data[18];
+	uint8_t data[18];
 };
 
 struct icx3_info {
-	unsigned char length;
+	uint8_t length;
-	unsigned char reserved;
+	uint8_t reserved;
-	unsigned char slave_address;
+	uint8_t slave_address;
-	unsigned char product_id;
+	uint8_t product_id;
-	unsigned char major_version;
+	uint8_t major_version;
-	unsigned char minor_version;
+	uint8_t minor_version;
 };
 
 enum icx3_product_id {

nvidia-dcgm.c

@@ -0,0 +1,130 @@
+#include <stdio.h>
+
+#include "nvidia-dcgm.h"
+
+/* Yeah I know this is duplicated */
+#define HEADER_COLOR_START "\x1b[36m"
+#define HEADER_COLOR_END "\x1b[39m"
+
+dcgmHandle_t 	dcgm_handle = (dcgmHandle_t)NULL;
+dcgmFieldGrp_t 	field_group_id;
+
+unsigned long long clock_reasons = 0;
+
+void init_dcgm()
+{
+	/* Init DCGM */
+	dcgmReturn_t result;
+	result = dcgmInit();
+	if (result != DCGM_ST_OK)
+		printf("Could not init DCGM: %s\n", errorString(result));
+	
+	/* Start embedded */
+	result = dcgmStartEmbedded(DCGM_OPERATION_MODE_MANUAL, &dcgm_handle);
+	if (result != DCGM_ST_OK)
+		printf("Could not start embedded DCGM engine: %s\n", errorString(result));
+	
+	/* Create metric group */
+	int num_metrics = sizeof(metric_readings) / sizeof(metric_readings[0]);
+	unsigned short field_ids[num_metrics + 1];
+	for (int i = 0; i < num_metrics; i++) {
+		field_ids[i] = metric_readings[i].field_id;
+		metric_readings[i].last_reading = 0.0;
+	}
+	field_ids[num_metrics] = DCGM_FI_DEV_CLOCKS_EVENT_REASONS;
+	result = dcgmFieldGroupCreate(dcgm_handle, num_metrics + 1, &field_ids[0], (char *)"evga_icx_fields", &field_group_id);
+	if (result != DCGM_ST_OK)
+		printf("Could not create DCGM metric group: %s\n", errorString(result));
+	
+	/* Start recording */
+	result = dcgmWatchFields(dcgm_handle, DCGM_GROUP_ALL_GPUS , field_group_id, 1000000, 30, 1);
+	if (result != DCGM_ST_OK)
+		printf("Could not record DCGM metrics: %s\n", errorString(result));
+}
+
+void shutdown_dcgm()
+{
+	dcgmStopEmbedded(dcgm_handle);
+	dcgmShutdown();
+}
+
+void print_dcgm(int compact, int overwrite)
+{
+	dcgmReturn_t result;
+	dcgmUpdateAllFields(dcgm_handle, 1);
+	result = dcgmGetLatestValues(dcgm_handle, DCGM_GROUP_ALL_GPUS, field_group_id, &update_field_values, NULL);
+	if (result != DCGM_ST_OK)
+		printf("Could not read DCGM metrics: %s\n", errorString(result));
+	
+	/* Print the float metrics */
+	for (int i = 0; i < sizeof(metric_readings) / sizeof(metric_readings[0]); i++) {
+		if (compact)
+			printf("%s%s%s%s %4.1f",
+				i==0 ? "": " ",
+				HEADER_COLOR_START,
+				metric_readings[i].short_name,
+				HEADER_COLOR_END,
+				metric_readings[i].last_reading * 100);
+		else
+			printf("%s: %.1f%\n", metric_readings[i].long_name, metric_readings[i].last_reading * 100);
+	}
+	if (compact)
+		printf("%%");
+	
+	/* Print clock reasons */
+	int single_reason = 1;
+	if (compact)
+		printf("%s CLK %s", HEADER_COLOR_START, HEADER_COLOR_END);
+	else
+		printf("Clock reasons: ");
+	
+	for (int i = 0; i < (sizeof(clock_reason_names) / sizeof(struct clock_reason)); i++) {
+		if (clock_reasons & clock_reason_names[i].mask) {
+			if (!single_reason) {
+				if (compact)
+					printf(",");
+				else
+					printf(", ");
+			}	
+			single_reason = 0;
+			
+			if (compact)
+				printf("%s", clock_reason_names[i].short_name);
+			else
+				printf("%s", clock_reason_names[i].long_name);
+		}
+	}
+	
+	if (single_reason)
+		printf("None");
+	
+	if (!compact)
+		printf(" (0x%llx)", clock_reasons);
+	else
+		printf("%-15s", " ");
+	
+	if (overwrite)
+		printf("\x1b[1G");
+}
+
+int update_field_values(unsigned int gpu_id, dcgmFieldValue_v1 *values, int num_values, void *userdata)
+{
+	for (int j = 0; j < num_values; j++) {
+		if (values[j].fieldType == DCGM_FT_DOUBLE) {
+			/* A double reading so read it into our readings array */
+			for (int i = 0; i < sizeof(metric_readings) / sizeof(metric_readings[0]); i++) {
+				if (metric_readings[i].field_id == values[j].fieldId) {
+					double val = values[j].value.dbl;
+					if (val >= 1)
+						metric_readings[i].last_reading = 0.0;
+					else
+						metric_readings[i].last_reading = values[j].value.dbl;
+				}
+			}
+		} else if (values[j].fieldType == DCGM_FT_INT64) {
+			/* Int is probably the bitmask of our clock reasons */
+			clock_reasons = values[j].value.i64;
+		}
+	}
+	return 0;
+}

nvidia-dcgm.h

@@ -0,0 +1,51 @@
+#include <dcgm_agent.h>
+#include <dcgm_structs.h>
+
+#include "evga-card.h"
+
+struct metric_reading {
+	unsigned short field_id;
+	char *short_name;
+	char *long_name;
+	double last_reading;
+};
+
+static struct metric_reading metric_readings[] =
+{
+	{DCGM_FI_PROF_GR_ENGINE_ACTIVE,			"GFX",		"Graphics Engine Activity"},
+	{DCGM_FI_PROF_SM_ACTIVE,				"SM ACT",	"SM Activity"},
+	{DCGM_FI_PROF_SM_OCCUPANCY,				"OCC",		"SM Occupancy"},
+	{DCGM_FI_PROF_PIPE_FP16_ACTIVE,			"FP16",		"FP16 Engine Activity"},
+	{DCGM_FI_PROF_PIPE_FP32_ACTIVE,			"FP32",		"FP32 Engine Activity"},
+	{DCGM_FI_PROF_PIPE_FP64_ACTIVE,			"FP64",		"FP64 Engine Activity"},
+	{DCGM_FI_PROF_PIPE_INT_ACTIVE,			"INT",		"Integer Activity"},
+	{DCGM_FI_PROF_PIPE_TENSOR_ACTIVE,		"TENS",		"Tensor Activity"},
+	{DCGM_FI_PROF_PIPE_TENSOR_HMMA_ACTIVE,	"HMMA",		"Half Precision Tensor"},
+	{DCGM_FI_PROF_PIPE_TENSOR_IMMA_ACTIVE,	"IMMA",		"Integer Tensor"},
+	{DCGM_FI_PROF_PIPE_TENSOR_DFMA_ACTIVE,	"DFMA",		"Double Precision Tensor"},
+	{DCGM_FI_PROF_DRAM_ACTIVE,				"DRAM",		"Memory BW Utilization"}
+};
+
+struct clock_reason {
+	unsigned long long mask;
+	char *short_name;
+	char *long_name;
+};
+
+static struct clock_reason clock_reason_names[] = 
+{
+	{DCGM_CLOCKS_EVENT_REASON_GPU_IDLE,			"Idle", 	"GPU idle"},
+	{DCGM_CLOCKS_EVENT_REASON_CLOCKS_SETTING,	"AppClk",	"Application clocks"},
+	{DCGM_CLOCKS_EVENT_REASON_SW_POWER_CAP,		"Pwr",		"Power cap"},
+	{DCGM_CLOCKS_EVENT_REASON_HW_SLOWDOWN,		"HWSlow",	"Hardware slowdown"},
+	{DCGM_CLOCKS_EVENT_REASON_SYNC_BOOST,		"Sync",		"Sync boost"},
+	{DCGM_CLOCKS_EVENT_REASON_SW_THERMAL,		"SWTherm",	"Software thermal"},
+	{DCGM_CLOCKS_EVENT_REASON_HW_THERMAL,		"HWTherm",	"Hardware thermal"},
+	{DCGM_CLOCKS_EVENT_REASON_HW_POWER_BRAKE,	"HWPower",	"Hardware power brake"},
+	{DCGM_CLOCKS_EVENT_REASON_DISPLAY_CLOCKS,	"DispClk",	"Display clock"}
+};
+
+void init_dcgm();
+void shutdown_dcgm();
+void print_dcgm(int compact, int overwrite);
+int update_field_values(unsigned int gpu_id, dcgmFieldValue_v1 *values, int num_values, void *userdata);

nvidia-sensors.c

@@ -47,6 +47,8 @@ void print_nvml_clock_reason(int compact, struct card_info *card)
 	
 	if (!compact)
 		printf(" (0x%llx)", reasons);
+	else
+		printf("%-15s", " ");
 }
 
 unsigned int get_nvml_temp(struct card_info *card)