Documentation

NVML temp code updated (driver version ~555 and up)

Makefile

@@ -1,29 +1,29 @@
 .PHONY : clean debug
 
-OBJS = evga-icx.o evga-card.o icx3.o
+OBJS = blackwell-sensors.o card.o evga-icx.o evga-card.o gddr6.o icx3.o board-sensors.o zen3-rapl.o
-LDLIBS = -li2c
+LDLIBS = -li2c -lm
-CFLAGS = -MD
+CFLAGS = -MD -I/usr/local/cuda/targets/x86_64-linux/include/
 
 ifdef USE_NVML
-LDLIBS += -lnvidia-ml
+LDLIBS += -lnvidia-ml -L/usr/local/cuda/targets/x86_64-linux/lib/stubs/
 CFLAGS += -DUSE_NVML
 OBJS += nvidia-sensors.o
 endif
 
-ifdef USE_LIBPCI
+ifdef USE_DCGM
-LDLIBS += -lpci
+LDLIBS += -ldcgm
-CFLAGS += -DUSE_LIBPCI
+CFLAGS += -DUSE_DCGM -I/usr/include/datacenter-gpu-manager-4/
-OBJS += gddr6.o
+OBJS += nvidia-dcgm.o
 endif
 
 evga-icx : $(OBJS)
 
-debug : CFLAGS += -g -O0
+debug : CFLAGS += -g -Og
 debug : evga-icx
 
 clean :
-	rm evga-icx
 	rm *.o
 	rm *.d
+	rm evga-icx
 	
 -include $(OBJS:.o=.d)

README.md

@@ -2,8 +2,10 @@
 
 This program allows you to read temperature sensors off of supported EVGA 30-series iCX3 video cards, as well as control the fans individually.
 
+Also a mashup of a bunch of other temperature sensor stuff too like Blackwell cards, lm_sensors, and DCGM metrics that I have just thrown in here.
+
 ## Prerequisites
-A supported EVGA 30-series iCX3 card.  I have not done extensive testing but belive this is every model of their:
+A supported EVGA 30-series card with iCX3.  This includes:
 * RTX 3060 Ti
 * RTX 3070
 * RTX 3070 Ti
@@ -14,6 +16,8 @@ A supported EVGA 30-series iCX3 card.  I have not done extensive testing but bel
 
 The number of fans supported depends, of course, on your particular model.
 
+You must have the `i2c-dev` kernel module loaded with `modprobe i2c-dev`
+
 Access to the `/dev/i2c` device files, which means either:
 * Run as root, or
 * Install udev rules to allow user access. If you have the OpenRGB udev rules installed to control the LEDs you already have this set up.
@@ -21,7 +25,7 @@ Access to the `/dev/i2c` device files, which means either:
 ## Dependencies
 * libi2c-dev
 * libnvidia-ml-dev (if building with `USE_NVML=1`)
-* libpci-dev (if building with `USE_LIBPCI=1`)
+* datacenter-gpu-manager-4-dev (if building with `USE_DCGM=1`)
 
 ## Building
 `make`
@@ -32,13 +36,20 @@ Access to the `/dev/i2c` device files, which means either:
 Add the make flag `USE_NVML=1` and the it will also display the main GPU temperature ("GPU1") as reported by the NVIDIA driver.  It will also display the performance cap/clock reason and memory controller utilization.  This requires the NVIDIA management library (NVML) to be installed.
 
 ### VRAM and Hotspot temperature
-Add the make flag `USE_LIBPCI=1` and you can also read the VRAM and "hotspot" temperatures.  These require direct memory access to the PCI device so you must run as root and also enable the kernel parameter `iomem=relaxed`.  These sensors are **extremely** undocumented so I can't say anything about their accuracy.
+Run as root and you can also read the VRAM and "hotspot" temperatures.  These require access the BAR0 through the `/sys/bus/pci/devices/*/resource0` device files.  These sensors are **extremely** undocumented so I can't say anything about their accuracy. If you have a Blackwell (50 series) card you can read all 4 die temp sensors, a 'system' temperature, and every individual VRAM chip temp (add your card in blackwell-temps.c). You will probably require the kernel parameter `iomem=relaxed`.
+
+### Datacenter GPU Manager metrics
+Add the make flag `USE_DCGM=1` and it can display real-time metrics from supported Nvidia GPUs like SM occupancy, integer/fp pipe usage, tensor usage, and DRAM bandwidth utilization. This requires a Quadro/Workstation or a Tesla/Datacenter card, it does not work on GeForce cards.
+
+### Hardware monitoring
+This program can also monitor hardware sensors using the standard linux `hwmon` API.  See board-sensors.h for examples of how to configure these, they must be staticlly compiled in.
 
 ## Usage
 Note that when controlling fans directly through iCX3 they will fall offline from the Nvidia driver and show as 0 RPM until you return them to automatic mode.
 
 ```text
 Available options:
+--i2c N      : Only probe I2C bus N instead of all (may help with stuttering or freezing when probing I2C devices)
 --gpu N      : Control only GPU N instead of all supported cards
 --fan SPEED  : Set all fans at once to SPEED (see below)
 --fanN SPEED : Set fan N (0-3) to SPEED
@@ -52,6 +63,8 @@ Available options:
 --watch N    : Keep printing output every N seconds
 --overwrite  : Overwrite previously displayed info with --watch and --compact instead of continuously logging new lines
 --color      : Print headers in color in --compact mode for better readability
+--no-reasons : Do not query NVML for clock reasons (can cause stuttering)
+--board      : Also print temperatures from the CPU, motherboard, and other sensors
 ```
 
 ### Examples:

blackwell-sensors.c

@@ -0,0 +1,188 @@
+#include <stdio.h>
+#include <string.h>
+#include <dirent.h>
+#include <stdint.h>
+#include <math.h>
+#include <stdbool.h>
+
+#include "blackwell-sensors.h"
+
+/* Number of die sensors, 4 for Blackwell */
+#define BLACKWELL_NUM_DIE_SENSORS 4
+/* Memory spacing between DRAM FBPA blocks */
+#define BLACKWELL_FBPA_STRIDE 0x4000
+/* Maximum count of FBPA blocks, technically up to 18 but no consumer card has more than 8 */
+#define BLACKWELL_MAX_NUM_FBPA 9
+
+/* Yeah I know this is duplicated */
+#define HEADER_COLOR_START "\x1b[36m"
+#define HEADER_COLOR_END "\x1b[39m"
+
+static float read_temp_sensor(void *addr);
+static int read_dram_temp_sensor(void *addr);
+
+struct blackwell_card_info {
+	char *card_name;
+	unsigned short vendor_id;
+	unsigned short device_id;
+	int die_temp_offset[BLACKWELL_NUM_DIE_SENSORS];
+	int sys_temp_offset;
+	int dram_temp_offset_start;
+	int clamshell;			/* If the card has dual-side ram, if not then half the DRAM temps are repeated */
+};
+
+static struct blackwell_card_info blackwell_card_list[] =
+{
+	{"NVIDIA RTX PRO 6000 Blackwell Workstation Edition",	NVIDIA_VEN,		NVIDIA_RTXPRO6000_WORKSTATION_DEV,	{0xAD0A90, 0xAD0A94, 0xAD0A98, 0xAD0A9C},	0xAD06EC,	0x9024C0,	true}
+};
+
+
+/* Search all i2c device files for ones are on a PCI device of a supported GPU,
+   and respond with the correct iCX3 version information */
+int find_blackwell_gpus(struct card_info *infos, int num_gpus, int max_gpus)
+{
+	char pci_devices_path[PATH_MAX];
+	char device_path[PATH_MAX];
+	char dev_file[PATH_MAX];
+	char pci_path[PATH_MAX];
+	char *pci_addr;
+
+	DIR *dir;
+	struct dirent *ent;
+	
+	unsigned short pci_vendor, pci_device = 0;
+	
+	/* Start looking for PCI */
+	strcpy(pci_devices_path, "/sys/bus/pci/devices/");
+	dir = opendir(pci_devices_path);
+	
+	/* make sure we can open the PCI device */
+	if(dir == NULL)
+		return num_gpus;
+	
+	/* loop over all PCI devices */
+	while((ent = readdir(dir)) != NULL)
+	{				
+		strcpy(device_path, pci_devices_path);
+		strcat(device_path, ent->d_name);
+						
+		pci_vendor = read_pci_id(device_path, "/vendor");
+		pci_device = read_pci_id(device_path, "/device");
+		
+		/* See if it's a supported Blackwell card*/
+		for (int i = 0; i < (sizeof(blackwell_card_list) / sizeof(struct blackwell_card_info)); i++) {
+			
+			if (pci_vendor == blackwell_card_list[i].vendor_id &&
+			    pci_device == blackwell_card_list[i].device_id) {
+				
+				infos[num_gpus].card_name     = blackwell_card_list[i].card_name;
+				infos[num_gpus].pci_id        = strdup(ent->d_name);
+				infos[num_gpus].pci_dev_path  = strdup(device_path);
+				infos[num_gpus].pci_device_id = pci_device;
+				infos[num_gpus].i2c_dev_path  = NULL;
+				infos[num_gpus].i2c_fd        = -1;
+				
+				mmap_bar0(&infos[num_gpus]);
+				num_gpus++;
+				break;
+			}
+		}
+		
+		if (num_gpus == max_gpus)
+			break;
+	}
+	
+	return num_gpus;
+}
+
+void print_blackwell_info(int compact, int overwrite, struct card_info *gpu)
+{
+	if (gpu->bar0_mmap == NULL)
+		return;
+	
+	for (int i = 0; i < (sizeof(blackwell_card_list) / sizeof(struct blackwell_card_info)); i++) {
+		if (!compact)
+			printf("%s @ %s\n", gpu->card_name, gpu->pci_id);
+		float temp;
+		
+		if (blackwell_card_list[i].device_id == gpu->pci_device_id) {
+			/* Print die sensors */
+			if (compact)
+				printf("%sDIE%s", HEADER_COLOR_START, HEADER_COLOR_END);
+			for (int die_sens_num = 0; die_sens_num < BLACKWELL_NUM_DIE_SENSORS; die_sens_num++) {
+				temp = read_temp_sensor(gpu->bar0_mmap + blackwell_card_list[i].die_temp_offset[die_sens_num]);
+				if (compact)
+					printf(" %3.0f", temp);
+				else
+					printf("Die %d: %+.1f°C\n", die_sens_num, temp);
+			}
+			
+			/* Sys temp */
+			temp = read_temp_sensor(gpu->bar0_mmap + blackwell_card_list[i].sys_temp_offset);
+			if (compact)
+				printf(" %sSYS%s %3.0f", HEADER_COLOR_START, temp, HEADER_COLOR_END);
+			else
+				printf("System: %+.1f°C\n", temp);
+			
+			/* DRAM temps */
+			int dram_num = 0;
+			int num_channels = blackwell_card_list[i].clamshell ? 4 : 2;
+			int dram_max = 0;
+			int dram_min = 120;
+			int dram_total = 0.0;
+			for (int fbpa_num = 0; fbpa_num < BLACKWELL_MAX_NUM_FBPA; fbpa_num++) {
+				for (int chan_num = 0; chan_num < num_channels; chan_num++) {
+					temp = read_dram_temp_sensor(gpu->bar0_mmap + 
+						blackwell_card_list[i].dram_temp_offset_start + 
+						BLACKWELL_FBPA_STRIDE * fbpa_num +
+						0x4 * chan_num + 
+						2);
+						
+					if (temp == 0.0)
+						break;
+					
+					dram_max = (temp > dram_max) ? temp : dram_max;
+					dram_min = (temp < dram_min) ? temp : dram_min;
+					dram_total += temp;
+					
+					if (!compact)
+						printf("DRAM %i: %+.0f°C\n", dram_num, temp);
+					dram_num++;
+				}
+			}
+			if (compact) {
+				printf(" %sDRAM HI%s %3.0f %sAVG%s %5.1f %sLO%s %3.0f",
+					HEADER_COLOR_START,
+					HEADER_COLOR_END,
+					(float)dram_max,
+					HEADER_COLOR_START,
+					HEADER_COLOR_END,
+					(float)dram_total/dram_num,
+					HEADER_COLOR_START,
+					HEADER_COLOR_END,
+					(float)dram_min);
+			} else {
+				printf("DRAM average: %+.1f°C\n", (float)dram_total/dram_num);
+				printf("DRAM min: %+.0f°C\n", (float)dram_min);
+				printf("DRAM max: %+.0f°C\n", (float)dram_max);
+			}
+			return;
+		}
+	}
+}
+
+float read_temp_sensor(void *addr)
+{
+	return (*(uint16_t *)addr) / (float)0x100;
+}
+
+int read_dram_temp_sensor(void *addr)
+{
+	uint16_t raw = *(uint16_t *)addr;
+	if (raw == 0xBADF)
+		return 0.0;
+	
+	int code = (raw >> 8) & 0xFF;
+	if (code > 80) code = 80;
+	return (code > 19) ? (code - 20) * 2 : -(40 - code * 2);
+}

blackwell-sensors.h

@@ -0,0 +1,10 @@
+#ifndef BLACKWELL_SENSORS_H
+#define BLACKWELL_SENSORS_H
+
+#include "card.h"
+
+
+
+int find_blackwell_gpus(struct card_info *infos, int num_gpus, int max_gpus);
+void print_blackwell_info(int compact, int overwrite, struct card_info *gpu);
+#endif

board-sensors.c

@@ -0,0 +1,122 @@
+#include <dirent.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "board-sensors.h"
+
+int find_board_sensors(struct hwmon_avail_sensor *board_sensors, int max_sensors)
+{
+	const char *hwmon_path = "/sys/class/hwmon/";
+	char device_path[NAME_MAX];
+	char sensor_path[NAME_MAX];
+	
+	char driver_name[256];
+	
+	FILE *file;
+	DIR *dir;
+	struct dirent *ent;
+	
+	int num_sensors = 0;
+	
+	/* Start looking for hwmon devices in /sys/class/hwmon/ */
+	dir = opendir(hwmon_path);
+	
+	/* make sure we can open the device directory */
+	if(dir == NULL)
+		return 0;
+	
+	/* loop over all hwmon devices */
+	while((ent = readdir(dir)) != NULL)
+	{
+		/* Ignore any non-hwmon dirs */
+		if(strncmp(ent->d_name, "hwmon", 5) != 0) 
+			continue;
+		
+		strcpy(device_path, hwmon_path);
+		strcat(device_path, ent->d_name);
+		
+		/* Read in the name of the device */
+		strcpy(sensor_path, device_path);
+		strcat(sensor_path, "/name");
+		
+		file = fopen(sensor_path, "r");
+		if (file == NULL)
+			continue;
+		
+		if (fgets(driver_name, sizeof(driver_name), file) == NULL) {
+			fclose(file);
+			continue;
+		}
+		
+		fclose(file);
+
+		/* Driver names have a linebreak at the end so let's remove that for comparison*/
+		driver_name[strlen(driver_name) - 1] = '\0';
+		
+		/* Loop through all supported sensors and see if any are present in this device */
+		for (int i = 0; i < (sizeof(hwmon_sensor_info) / sizeof(struct hwmon_sensor)); i++) {
+			if (strcmp(driver_name, hwmon_sensor_info[i].driver_name) == 0) {
+				/* We matched the driver name, try to open the files */
+				strcpy(sensor_path, device_path);
+				strcat(sensor_path, "/");
+				strcat(sensor_path, hwmon_sensor_info[i].sensor_file_name);
+				strcat(sensor_path, "_input");
+				
+				file = fopen(sensor_path, "r");
+				if (file != NULL) {
+					fclose(file);
+					/* Good open of the sensor file */
+					board_sensors[num_sensors].file = calloc(NAME_MAX, sizeof(char));
+					strcpy(board_sensors[num_sensors].file, sensor_path);
+					board_sensors[num_sensors].sort_index = i;
+					board_sensors[num_sensors].sensor_info = &hwmon_sensor_info[i];
+					
+					/* Read in the sensor name */
+					board_sensors[num_sensors].sensor_name = calloc(MAX_SENSOR_NAME_LENGTH, sizeof(char));
+					strcpy(sensor_path, device_path);
+					strcat(sensor_path, "/");
+					strcat(sensor_path, hwmon_sensor_info[i].sensor_file_name);
+					strcat(sensor_path, "_label");
+					
+					file = fopen(sensor_path, "r");
+					if (file != NULL)
+						fgets(board_sensors[num_sensors].sensor_name, MAX_SENSOR_NAME_LENGTH, file);
+
+					/* Sensor name seems to always have a trailing newline we don't want */
+					size_t len_without_newline = strcspn(board_sensors[num_sensors].sensor_name, "\n");
+					board_sensors[num_sensors].sensor_name[len_without_newline] = '\0';
+					
+					if (num_sensors == max_sensors)
+						return num_sensors;
+					num_sensors++;
+				}
+			}
+		}
+		
+	}
+	
+	return num_sensors;
+}
+
+/* Returns 0 on a bad read or missing sensor, 1 on OK */
+int get_sensor_reading(struct hwmon_avail_sensor *sensor, float *reading) {
+	char buf[256] = {0};
+	long int raw;
+	FILE *file;
+	
+	file = fopen(sensor->file, "r");
+	if (file == NULL)
+		return 0;
+
+	fgets(buf, 256, file);
+	raw = strtol(buf, NULL, 10);
+	
+	fclose(file);
+	
+	*reading = (float)raw / sensor->sensor_info->divisor;
+
+	if (*reading == sensor->sensor_info->bad_value)
+		return 0;
+	
+	return 1;
+}

board-sensors.h

@@ -0,0 +1,61 @@
+#ifndef BOARD_SENSORS_H
+#define BOARD_SENSORS_H
+
+#include <stdio.h>
+
+#define MAX_SENSOR_NAME_LENGTH 256
+
+struct hwmon_sensor {
+	char *driver_name;				/* Contents of /sys/class/hwmon/hwmonX/name */
+	char *sensor_file_name;			/* Sysfs file to read */
+	char *name_prefix;				/* Prefix to attach to temp*_label for clarity */
+	char *short_name;				/* 'Category' name when using compact mode */
+	char *units;					/* Units string */
+	float divisor;					/* Divisor to convert temp* to units */
+	float bad_value;	/* Raw value that indicates a bad (missing) sensor TODO: verfiy most of these*/
+};
+
+/* Note the order here matters, it's the order these will be printed in */
+static struct hwmon_sensor hwmon_sensor_info[] = 
+{
+	{"zenpower",		"temp1",	"CPU ",			"CPU",	"°C",	1000.0,	-40.0	},	/* Tdie */
+	{"asusec",			"temp2",	"Motherboard ",	"CPU",	"°C",	1000.0,	-40.0	},	/* CPU */
+	{"zenpower",		"temp3",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd1 */
+	{"zenpower",		"temp4",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd2 */
+	{"zenpower",		"temp5",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd3 */
+	{"zenpower",		"temp6",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd4 */
+	{"zenpower",		"temp7",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd5 */
+	{"zenpower",		"temp8",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd6 */
+	{"zenpower",		"temp9",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd7 */
+	{"zenpower",		"temp10",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd8 */
+	{"zen-rapl",		"",			"CPU ",			"POW",	" W",	0.0,	0.0		},	/* Zen RAPL placeholder */
+	{"asusec",			"temp1",	"Motherboard ",	"CHIP",	"°C",	1000.0,	-40.0	},	/* Chipset */
+	{"asusec",			"temp5",	"Motherboard ",	"VRM",	"°C",	1000.0,	-40.0	},	/* VRM */
+	{"asusec",			"temp3",	"",				"MOBO",	"°C",	1000.0,	-40.0	},	/* Motherboard */
+	/*{"r8169_0_500:00",	"temp1",	"Ethernet",		"ETH",	"°C",	1000.0,	-40.0	},	/* Ethernet adapter */
+	{"nvme",			"temp1",	"NVMe ",		"NVME",	"°C",	1000.0,	-40.0	},	/* NVME Composite */
+	{"asusec",			"temp6",	"Motherboard ",	"AIR",	"°C",	1000.0,	-40.0	},	/* Water_In */
+	{"asusec",			"temp4",	"Motherboard ",	"AIR",	"°C",	1000.0,	-40.0	},	/* T_Sensor */
+	{"asusec",			"temp7",	"Motherboard ",	"AIR",	"°C",	1000.0,	-40.0	},	/* Water_Out */
+	{"nct6798",			"fan2",		"CPU fan",		"CPU",	"%",	15.0,	0.0		},	/* cpu_fan, cpu mid */
+	{"asusec",			"fan1",		"",				"CPU",	"%",	15.0,	0.0		},	/* cpu_opt, cpu front */
+	{"nct6798",			"fan5",		"H amp",		"CHA",	"%",	12.0,	0.0		},	/* h_amp, front fan */
+	{"nct6798",			"fan3",		"Chassis 2",	"CHA",	"%",	12.0,	0.0		},	/* cha2, top front */
+	{"nct6798",			"fan1",		"Chassis 1",	"CHA",	"%",	12.0,	0.0		},	/* cha1, top rear */
+	{"nct6798",			"fan4",		"Chassis 3",	"CHA",	"%",	13.0,	0.0		},	/* cha3, rear */
+	{"nct6798",			"fan6",		"AIO pump",		"CHA",	"%",	12.0,	0.0		},	/* aio_pump? */
+	{"nct6798",			"fan7",		"W pump",		"CHA",	"%",	12.0,	0.0		},	/* w_pump+? */
+	{"asusec",			"fan2",		"",				"CHIP",	"%",	35.0,	0.0		},	/* chipset */
+};
+
+struct hwmon_avail_sensor {
+	char *sensor_name;					/* Sensor name as read from the sysfs file */
+	int sort_index;						/* Sort index for order to display in */
+	char *file;							/* File to read from */
+	struct hwmon_sensor *sensor_info;	/* Associated sensor info struct */
+};
+
+int find_board_sensors(struct hwmon_avail_sensor *board_sensors, int max_sensors);
+int get_sensor_reading(struct hwmon_avail_sensor *sensor, float *reading);
+
+#endif

card.c

@@ -0,0 +1,96 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+#include "card.h"
+
+/* Read a 16-bit unsigned int from a pci id file */
+unsigned short read_pci_id(char *device_path, char *field)
+{
+	char buf[16];
+	char file_path[PATH_MAX];
+	
+	strcpy(file_path, device_path);
+	strcat(file_path, field);
+	
+	FILE *fp = fopen(file_path, "r");
+	
+	if (fp == NULL)
+		return 0;
+		
+	if (fgets(buf, sizeof(buf), fp) == NULL) {
+		fclose(fp);
+		return 0;
+	}
+	
+	fclose(fp);
+	
+	return (unsigned short)strtoul(buf, NULL, 16);
+}
+
+/* Read a PCI ID from the name of an Nvidia i2c device */
+char *read_nvidia_pci_address(char *device_path)
+{
+	char file_path[PATH_MAX];
+	
+	char *ret = calloc(16 + 1, sizeof(char)); /* assuming pci ids could look as large as 00000000:0C:00.0 */
+	
+	strcpy(file_path, device_path);
+	strcat(file_path, "/name");
+	
+	FILE *fp = fopen(file_path, "r");
+	
+	if (fp == NULL) {
+		free(ret);
+		return NULL;
+	}
+	
+	fscanf(fp, "NVIDIA i2c adapter %*u at %16s", ret);
+	fclose(fp);
+	
+	if (strlen(ret) == 0) {
+		free(ret);
+		return NULL;
+	}
+	
+	return ret;
+}
+
+/* Open an mmap to the BAR0 of a PCI device using its /sys/bus/pci device files */
+void mmap_bar0(struct card_info *card)
+{
+	char resource_path[PATH_MAX];
+	snprintf(resource_path, sizeof(resource_path), "%s/resource0", card->pci_dev_path);
+	
+	/* Open FD on the resource file */
+	int fd = open(resource_path, O_RDONLY);
+	if (fd == -1) {
+		printf("Unable to open %s\n", resource_path);
+		card->bar0_mmap = NULL;
+		return;
+	}
+
+	/* Figure out the size of BAR0 fomr the size of the resource file */
+	struct stat st;
+	if (fstat(fd, &st) == -1) {
+		printf("Unable to stat %s\n", resource_path);
+		close(fd);
+		card->bar0_mmap = NULL;
+		return;
+	}
+
+	/* mmap the bar0 resource file */
+	card->bar0_mmap = mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
+	if (card->bar0_mmap == MAP_FAILED) {
+		printf("Unable to mmap %s\n", resource_path);
+		card->bar0_mmap = NULL;
+	}
+
+	/* mmap stays open after the FD has been closed */
+	close(fd);
+}

card.h

@@ -0,0 +1,48 @@
+#ifndef CARD_H
+#define CARD_H
+
+#define NVIDIA_VEN                                      0x10DE
+
+#define NVIDIA_RTX3060_DEV                              0x2503
+#define NVIDIA_RTX3060_GA104_DEV                        0x2487
+#define NVIDIA_RTX3060_GA106_DEV                        0x2503
+#define NVIDIA_RTX3060_LHR_DEV                          0x2504
+#define NVIDIA_RTX3060TI_DEV                            0x2486
+#define NVIDIA_RTX3060TI_GDDR6X_DEV                     0x24C9
+#define NVIDIA_RTX3060TI_LHR_DEV                        0x2489
+#define NVIDIA_RTX3060TI_V1_LHR_DEV                     0x2414
+#define NVIDIA_RTX3070_DEV                              0x2484
+#define NVIDIA_RTX3070_LHR_DEV                          0x2488
+#define NVIDIA_RTX3070TI_DEV                            0x2482
+#define NVIDIA_RTX3070TI_GA102_DEV                      0x2207
+#define NVIDIA_RTX3080_DEV                              0x2206
+#define NVIDIA_RTX3080_LHR_DEV                          0x2216
+#define NVIDIA_RTX3080_12G_LHR_DEV                      0x220A
+#define NVIDIA_RTX3080TI_DEV                            0x2208
+#define NVIDIA_RTX3090_DEV                              0x2204
+#define NVIDIA_RTX3090TI_DEV                            0x2203
+
+#define NVIDIA_RTXPRO6000_WORKSTATION_DEV				0x2BB1
+
+struct card_info {
+	char *card_name;					/* The 'nice' name of the card */
+	char *pci_id;						/* PCI bus address in domain:bus:device.function format. May be shortened (e.g. c:00.0) */
+	char *pci_dev_path;                 /* Path to the PCI device file */
+	void *bar0_mmap;                    /* Mmap of the PCI BAR0 */
+	unsigned short pci_device_id;		/* The device ID of the card, i.e. corresponds to the NVIDIA model number */
+	char *i2c_dev_path;					/* Path to the i2c device file */
+	int i2c_fd;							/* File descriptor for the i2c device file, for re-use */
+	int product_id;						/* EVGA internal product ID, as reported by the iCX3 controller */
+	void *nvml_device;					/* Pointer to nvmlDevice_t for use in NVML calls */
+};
+
+struct clock_reason {
+	unsigned long long mask;
+	char *short_name;
+	char *long_name;
+};
+
+unsigned short read_pci_id(char *device_path, char *field);
+char *read_nvidia_pci_address(char *device_path);
+void mmap_bar0(struct card_info *card);
+#endif

evga-card.c

@@ -6,13 +6,85 @@
 #include "evga-card.h"
 #include "icx3.h"
 
+struct gpu_pci_info {
+	char *card_name;
+	unsigned short vendor_id;
+	unsigned short device_id;
+	unsigned short subvendor_id;
+	unsigned short subdevice_id;
+};
+
+static struct gpu_pci_info evga_pci_ids[] =
+{
+	{"EVGA GeForce RTX 3060 Ti FTW3 Gaming"                , NVIDIA_VEN,   NVIDIA_RTX3060TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_GAMING_SUB_DEV            },
+	{"EVGA GeForce RTX 3060 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3060TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_ULTRA_SUB_DEV             },
+	{"EVGA GeForce RTX 3060 Ti FTW3 Ultra LHR"             , NVIDIA_VEN,   NVIDIA_RTX3060TI_LHR_DEV,   EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_ULTRA_KL_SUB_DEV          },
+	{"EVGA GeForce RTX 3070 Black Gaming"                  , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_BLACK_SUB_DEV                },
+	{"EVGA GeForce RTX 3070 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_GAMING_SUB_DEV               },
+	{"EVGA GeForce RTX 3070 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_SUB_DEV                },
+	{"EVGA GeForce RTX 3070 XC3 Ultra LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_LHR_SUB_DEV            },
+	{"EVGA GeForce RTX 3070 XC3 Ultra Gaming"              , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_GAMING_LHR_SUB_DEV     },
+	{"EVGA GeForce RTX 3070 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_FTW3_ULTRA_SUB_DEV               },
+	{"EVGA GeForce RTX 3070 FTW3 Ultra LHR"                , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_FTW3_ULTRA_LHR_SUB_DEV           },
+	{"EVGA GeForce RTX 3070 Ti XC3 Gaming"                 , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_GAMING_SUB_DEV             },
+	{"EVGA GeForce RTX 3070 Ti XC3 Ultra"                  , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_ULTRA_SUB_DEV              },
+	{"EVGA GeForce RTX 3070 Ti XC3 Ultra v2"               , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_ULTRA_V2_SUB_DEV           },
+	{"EVGA GeForce RTX 3070 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_FTW3_ULTRA_SUB_DEV             },
+	{"EVGA GeForce RTX 3070 Ti FTW3 Ultra v2"              , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_FTW3_ULTRA_V2_SUB_DEV          },
+	{"EVGA GeForce RTX 3080 XC3 Black"                     , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_BLACK_SUB_DEV                },
+	{"EVGA GeForce RTX 3080 XC3 Black LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_BLACK_LHR_SUB_DEV            },
+	{"EVGA GeForce RTX 3080 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_GAMING_SUB_DEV               },
+	{"EVGA GeForce RTX 3080 XC3 Gaming LHR"                , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_GAMING_LHR_SUB_DEV           },
+	{"EVGA GeForce RTX 3080 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_SUB_DEV                },
+	{"EVGA GeForce RTX 3080 XC3 Ultra LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_LHR_SUB_DEV            },
+	{"EVGA GeForce RTX 3080 XC3 Ultra Hybrid"              , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HYBRID_SUB_DEV         },
+	{"EVGA GeForce RTX 3080 XC3 Ultra Hybrid LHR"          , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HYBRID_LHR_SUB_DEV     },
+	{"EVGA GeForce RTX 3080 XC3 Ultra Hydro Copper"        , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HC_SUB_DEV             },
+	{"EVGA GeForce RTX 3080 FTW3 Gaming"                   , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_GAMING_SUB_DEV              },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_SUB_DEV               },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra v2 LHR"             , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_LHR_V2_SUB_DEV        },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra LHR"                , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_LHR_SUB_DEV           },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid"             , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_SUB_DEV        },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid LHR"         , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_LHR_SUB_DEV    },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid Gaming LHR"  , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_GAMING_SUB_DEV },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hydro Copper"       , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HC_SUB_DEV            },
+	{"EVGA GeForce RTX 3080 XC3 Ultra 12G"                 , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_XC3_ULTRA_SUB_DEV            },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra 12GB"               , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_FTW3_ULTRA_SUB_DEV           },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hydro Copper 12G"   , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_FTW3_ULTRA_HC_SUB_DEV        },
+	{"EVGA GeForce RTX 3080 Ti XC3 Gaming"                 , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_SUB_DEV             },
+	{"EVGA GeForce RTX 3080 Ti XC3 Ultra Gaming"           , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_ULTRA_GAMING_SUB_DEV       },
+	{"EVGA GeForce RTX 3080 Ti XC3 Gaming Hybrid"          , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_HYBRID_SUB_DEV      },
+	{"EVGA GeForce RTX 3080 Ti XC3 Gaming Hydro Copper"    , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_HC_SUB_DEV          },
+	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_SUB_DEV             },
+	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra Hybrid"          , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_HYBRID_SUB_DEV      },
+	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra Hydro Copper"    , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_HC_SUB_DEV          },
+	{"EVGA GeForce RTX 3090 XC3 Black"                     , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_BLACK_SUB_DEV                },
+	{"EVGA GeForce RTX 3090 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_GAMING_SUB_DEV               },
+	{"EVGA GeForce RTX 3090 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_SUB_DEV                },
+	{"EVGA GeForce RTX 3090 XC3 Ultra Hybrid"              , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_HYBRID_SUB_DEV         },
+	{"EVGA GeForce RTX 3090 XC3 Ultra Hydro Copper"        , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_HC_SUB_DEV             },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_SUB_DEV               },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra v2"                 , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_V2_SUB_DEV            },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra v3"                 , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_V3_SUB_DEV            },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra Hybrid"             , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_HYBRID_SUB_DEV        },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra Hydro Copper"       , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_HC_SUB_DEV            },
+	{"EVGA GeForce RTX 3090 K|NGP|N Hybrid"                , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_KINGPIN_HYBRID_SUB_DEV           },
+	{"EVGA GeForce RTX 3090 K|NGP|N Hydro Copper"          , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_KINGPIN_HC_SUB_DEV               },
+	{"EVGA GeForce RTX 3090 Ti FTW3 Black Gaming"          , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_BLACK_SUB_DEV             },
+	{"EVGA GeForce RTX 3090 Ti FTW3 Gaming"                , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_GAMING_SUB_DEV            },
+	{"EVGA GeForce RTX 3090 Ti FTW3 Ultra Gaming"          , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_ULTRA_GAMING_SUB_DEV      }
+};
+
+
+
 /* Search all i2c device files for ones are on a PCI device of a supported GPU,
    and respond with the correct iCX3 version information */
-int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
+int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus, int i2c_bus)
 {
-	char i2c_devices_path[NAME_MAX];
+	char i2c_devices_path[PATH_MAX];
-	char device_path[NAME_MAX];
+	char device_path[PATH_MAX];
-	char dev_file[NAME_MAX];
+	char dev_file[PATH_MAX];
+	char pci_path[PATH_MAX];
 	char *pci_addr;
 
 	FILE *test_fd;
@@ -20,6 +92,7 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
 	struct dirent *ent;
 	
 	int num_gpus = 0;
+	int current_i2c_bus = -1;
 	unsigned short pci_vendor, pci_device, pci_subsystem_vendor, pci_subsystem_device = 0;
 	
 	/* Start looking for I2C adapters in /sys/bus/i2c/devices/ */
@@ -37,6 +110,13 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
 		if(strncmp(ent->d_name, "i2c-", 4) != 0) 
 			continue;
 		
+		/* Only probe the specific device given (if provided) */
+		if (i2c_bus >= 0) {
+			sscanf(ent->d_name, "i2c-%i", &current_i2c_bus);
+			if (current_i2c_bus != i2c_bus)
+				continue;
+		}
+		
 		strcpy(device_path, i2c_devices_path);
 		strcat(device_path, ent->d_name);
 		
@@ -46,10 +126,14 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
 			continue;
 		
 		/* Read the PCI info for the underlying device */
-		pci_vendor           = read_pci_id(device_path, "/device/vendor");
+		strcat(device_path, "/../");
-		pci_device           = read_pci_id(device_path, "/device/device");
+		if (realpath(device_path, pci_path) == NULL)
-		pci_subsystem_vendor = read_pci_id(device_path, "/device/subsystem_vendor");
+			continue;
-		pci_subsystem_device = read_pci_id(device_path, "/device/subsystem_device");
+		
+		pci_vendor           = read_pci_id(pci_path, "/vendor");
+		pci_device           = read_pci_id(pci_path, "/device");
+		pci_subsystem_vendor = read_pci_id(pci_path, "/subsystem_vendor");
+		pci_subsystem_device = read_pci_id(pci_path, "/subsystem_device");
 		
 		/* See if it's a matching device for a supported EVGA card */
 		for (int i = 0; i < (sizeof(evga_pci_ids) / sizeof(struct gpu_pci_info)); i++) {
@@ -67,9 +151,10 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
 					/* Write our card info into the provided struct array */
 					infos[num_gpus].card_name = evga_pci_ids[i].card_name;
 					infos[num_gpus].pci_id = pci_addr;
+					infos[num_gpus].pci_dev_path = strdup(pci_path);
 					infos[num_gpus].pci_device_id = pci_device;
-					infos[num_gpus].i2c_dev_path = calloc(strlen(dev_file) + 1, sizeof(char));
+					infos[num_gpus].i2c_dev_path = strdup(dev_file);
-					strcpy(infos[num_gpus].i2c_dev_path, dev_file);
+					mmap_bar0(&infos[num_gpus]);
 					num_gpus++;
 					break;
 				}
@@ -78,59 +163,7 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
 		
 		if (num_gpus == max_gpus)
 			break;
-		
 	}
 	
 	return num_gpus;
 }
-
-/* Read a 16-bit unsigned int from a pci id file */
-unsigned short read_pci_id(char *device_path, char *field)
-{
-	char buf[16];
-	char file_path[NAME_MAX];
-	
-	strcpy(file_path, device_path);
-	strcat(file_path, field);
-	
-	FILE *fp = fopen(file_path, "r");
-	
-	if (fp == NULL)
-		return 0;
-		
-	if (fgets(buf, sizeof(buf), fp) == NULL) {
-		fclose(fp);
-		return 0;
-	}
-	
-	fclose(fp);
-	
-	return (unsigned short)strtoul(buf, NULL, 16);
-}
-
-char *read_nvidia_pci_address(char *device_path)
-{
-	char file_path[NAME_MAX];
-	
-	char *ret = calloc(16 + 1, sizeof(char)); /* assuming pci ids could look as large as 00000000:0C:00.0 */
-	
-	strcpy(file_path, device_path);
-	strcat(file_path, "/name");
-	
-	FILE *fp = fopen(file_path, "r");
-	
-	if (fp == NULL) {
-		free(ret);
-		return NULL;
-	}
-	
-	fscanf(fp, "NVIDIA i2c adapter %*u at %16s", ret);
-	fclose(fp);
-	
-	if (strlen(ret) == 0) {
-		free(ret);
-		return NULL;
-	}
-	
-	return ret;
-}

evga-card.h

@@ -1,26 +1,7 @@
 #ifndef EVGA_CARD_H
 #define EVGA_CARD_H
 
-#define NVIDIA_VEN                                      0x10DE
+#include "card.h"
-
-#define NVIDIA_RTX3060_DEV                              0x2503
-#define NVIDIA_RTX3060_GA104_DEV                        0x2487
-#define NVIDIA_RTX3060_GA106_DEV                        0x2503
-#define NVIDIA_RTX3060_LHR_DEV                          0x2504
-#define NVIDIA_RTX3060TI_DEV                            0x2486
-#define NVIDIA_RTX3060TI_GDDR6X_DEV                     0x24C9
-#define NVIDIA_RTX3060TI_LHR_DEV                        0x2489
-#define NVIDIA_RTX3060TI_V1_LHR_DEV                     0x2414
-#define NVIDIA_RTX3070_DEV                              0x2484
-#define NVIDIA_RTX3070_LHR_DEV                          0x2488
-#define NVIDIA_RTX3070TI_DEV                            0x2482
-#define NVIDIA_RTX3070TI_GA102_DEV                      0x2207
-#define NVIDIA_RTX3080_DEV                              0x2206
-#define NVIDIA_RTX3080_LHR_DEV                          0x2216
-#define NVIDIA_RTX3080_12G_LHR_DEV                      0x220A
-#define NVIDIA_RTX3080TI_DEV                            0x2208
-#define NVIDIA_RTX3090_DEV                              0x2204
-#define NVIDIA_RTX3090TI_DEV                            0x2203
 
 #define EVGA_SUB_VEN                                    0x3842
 
@@ -83,87 +64,6 @@
 #define EVGA_RTX3090TI_FTW3_GAMING_SUB_DEV              0x4983
 #define EVGA_RTX3090TI_FTW3_ULTRA_GAMING_SUB_DEV        0x4985
 
-struct card_info {
+int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus, int i2c_bus);
-	char *card_name;					/* The 'nice' name of the card */
-	char *pci_id;						/* PCI bus address in domain:bus:device.function format. May be shortened (e.g. c:00.0) */
-	unsigned short pci_device_id;		/* The device ID of the card, i.e. corresponds to the NVIDIA model number */
-	char *i2c_dev_path;					/* Path to the i2c device file */
-	int i2c_fd;							/* File descriptor for the i2c device file, for re-use */
-	int product_id;						/* EVGA internal product ID, as reported by the iCX3 controller */
-	unsigned int bar0;					/* Address of the card's PCI base address register */
-};
-
-struct gpu_pci_info {
-	char *card_name;
-	unsigned short vendor_id;
-	unsigned short device_id;
-	unsigned short subvendor_id;
-	unsigned short subdevice_id;
-};
-
-static struct gpu_pci_info evga_pci_ids[] =
-{
-	{"EVGA GeForce RTX 3060 Ti FTW3 Gaming"                , NVIDIA_VEN,   NVIDIA_RTX3060TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_GAMING_SUB_DEV            },
-	{"EVGA GeForce RTX 3060 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3060TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_ULTRA_SUB_DEV             },
-	{"EVGA GeForce RTX 3060 Ti FTW3 Ultra LHR"             , NVIDIA_VEN,   NVIDIA_RTX3060TI_LHR_DEV,   EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_ULTRA_KL_SUB_DEV          },
-	{"EVGA GeForce RTX 3070 Black Gaming"                  , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_BLACK_SUB_DEV                },
-	{"EVGA GeForce RTX 3070 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_GAMING_SUB_DEV               },
-	{"EVGA GeForce RTX 3070 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_SUB_DEV                },
-	{"EVGA GeForce RTX 3070 XC3 Ultra LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_LHR_SUB_DEV            },
-	{"EVGA GeForce RTX 3070 XC3 Ultra Gaming"              , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_GAMING_LHR_SUB_DEV     },
-	{"EVGA GeForce RTX 3070 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_FTW3_ULTRA_SUB_DEV               },
-	{"EVGA GeForce RTX 3070 FTW3 Ultra LHR"                , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_FTW3_ULTRA_LHR_SUB_DEV           },
-	{"EVGA GeForce RTX 3070 Ti XC3 Gaming"                 , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_GAMING_SUB_DEV             },
-	{"EVGA GeForce RTX 3070 Ti XC3 Ultra"                  , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_ULTRA_SUB_DEV              },
-	{"EVGA GeForce RTX 3070 Ti XC3 Ultra v2"               , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_ULTRA_V2_SUB_DEV           },
-	{"EVGA GeForce RTX 3070 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_FTW3_ULTRA_SUB_DEV             },
-	{"EVGA GeForce RTX 3070 Ti FTW3 Ultra v2"              , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_FTW3_ULTRA_V2_SUB_DEV          },
-	{"EVGA GeForce RTX 3080 XC3 Black"                     , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_BLACK_SUB_DEV                },
-	{"EVGA GeForce RTX 3080 XC3 Black LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_BLACK_LHR_SUB_DEV            },
-	{"EVGA GeForce RTX 3080 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_GAMING_SUB_DEV               },
-	{"EVGA GeForce RTX 3080 XC3 Gaming LHR"                , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_GAMING_LHR_SUB_DEV           },
-	{"EVGA GeForce RTX 3080 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_SUB_DEV                },
-	{"EVGA GeForce RTX 3080 XC3 Ultra LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_LHR_SUB_DEV            },
-	{"EVGA GeForce RTX 3080 XC3 Ultra Hybrid"              , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HYBRID_SUB_DEV         },
-	{"EVGA GeForce RTX 3080 XC3 Ultra Hybrid LHR"          , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HYBRID_LHR_SUB_DEV     },
-	{"EVGA GeForce RTX 3080 XC3 Ultra Hydro Copper"        , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HC_SUB_DEV             },
-	{"EVGA GeForce RTX 3080 FTW3 Gaming"                   , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_GAMING_SUB_DEV              },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_SUB_DEV               },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra v2 LHR"             , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_LHR_V2_SUB_DEV        },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra LHR"                , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_LHR_SUB_DEV           },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid"             , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_SUB_DEV        },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid LHR"         , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_LHR_SUB_DEV    },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid Gaming LHR"  , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_GAMING_SUB_DEV },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hydro Copper"       , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HC_SUB_DEV            },
-	{"EVGA GeForce RTX 3080 XC3 Ultra 12G"                 , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_XC3_ULTRA_SUB_DEV            },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra 12GB"               , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_FTW3_ULTRA_SUB_DEV           },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hydro Copper 12G"   , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_FTW3_ULTRA_HC_SUB_DEV        },
-	{"EVGA GeForce RTX 3080 Ti XC3 Gaming"                 , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_SUB_DEV             },
-	{"EVGA GeForce RTX 3080 Ti XC3 Ultra Gaming"           , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_ULTRA_GAMING_SUB_DEV       },
-	{"EVGA GeForce RTX 3080 Ti XC3 Gaming Hybrid"          , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_HYBRID_SUB_DEV      },
-	{"EVGA GeForce RTX 3080 Ti XC3 Gaming Hydro Copper"    , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_HC_SUB_DEV          },
-	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_SUB_DEV             },
-	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra Hybrid"          , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_HYBRID_SUB_DEV      },
-	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra Hydro Copper"    , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_HC_SUB_DEV          },
-	{"EVGA GeForce RTX 3090 XC3 Black"                     , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_BLACK_SUB_DEV                },
-	{"EVGA GeForce RTX 3090 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_GAMING_SUB_DEV               },
-	{"EVGA GeForce RTX 3090 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_SUB_DEV                },
-	{"EVGA GeForce RTX 3090 XC3 Ultra Hybrid"              , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_HYBRID_SUB_DEV         },
-	{"EVGA GeForce RTX 3090 XC3 Ultra Hydro Copper"        , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_HC_SUB_DEV             },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_SUB_DEV               },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra v2"                 , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_V2_SUB_DEV            },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra v3"                 , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_V3_SUB_DEV            },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra Hybrid"             , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_HYBRID_SUB_DEV        },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra Hydro Copper"       , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_HC_SUB_DEV            },
-	{"EVGA GeForce RTX 3090 K|NGP|N Hybrid"                , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_KINGPIN_HYBRID_SUB_DEV           },
-	{"EVGA GeForce RTX 3090 K|NGP|N Hydro Copper"          , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_KINGPIN_HC_SUB_DEV               },
-	{"EVGA GeForce RTX 3090 Ti FTW3 Black Gaming"          , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_BLACK_SUB_DEV             },
-	{"EVGA GeForce RTX 3090 Ti FTW3 Gaming"                , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_GAMING_SUB_DEV            },
-	{"EVGA GeForce RTX 3090 Ti FTW3 Ultra Gaming"          , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_ULTRA_GAMING_SUB_DEV      }
-};
-
-int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus);
-unsigned short read_pci_id(char *device_path, char *field);
-char *read_nvidia_pci_address(char *device_path);
 
 #endif

evga-icx.c

@@ -6,25 +6,35 @@
 #include <stdlib.h>
 
 #ifdef USE_NVML
-#include "nvidia-sensors.h"
+	#include "nvidia-sensors.h"
 #endif
 
-#ifdef USE_LIBPCI
+#ifdef USE_DCGM
+	#include "nvidia-dcgm.h"
+#endif
+
+#include "blackwell-sensors.h"
 #include "gddr6.h"
-#endif
-
 #include "icx3.h"
 #include "evga-card.h"
+#include "board-sensors.h"
+#include "zen3-rapl.h"
 
 #define MAX_GPUS 16
+#define MAX_BOARD_SENSORS 256
 #define HEADER_COLOR_START "\x1b[36m"
 #define HEADER_COLOR_END "\x1b[39m"
 
-char *header_start = "";
+static char *header_start = "";
-char *header_end = "";
+static char *header_end = "";
+
+static int zen3_rapl_sensor = -1;	/* Board sensor number for the RAPL sensor */
+static int compact = 0;				/* Compact one-line per GPU display */
+static int no_reasons = 0;			/* Don't probe or display NVML clock reasons */
 
 static const char helpstring[] = "Available options:\n"
-                                 "--gpu N      : Control only GPU N instead of all supported cards\n"
+								 "--i2c N      : Only probe I2C bus N instead of all (may help with stuttering or freezing when probing I2C devices)\n"
+								 "--gpu N      : Control only GPU N instead of all supported cards\n"
 								 "--fan SPEED  : Set all fans at once to SPEED (see below)\n"
 								 "--fanN SPEED : Set fan N (0-3) to SPEED\n"
 								 "    SPEED may be one of the following:\n"
@@ -36,24 +46,37 @@ static const char helpstring[] = "Available options:\n"
 								 "--compact    : Print sensor reading in a compact one-line per card format\n"
 								 "--watch N    : Keep printing output every N seconds\n"
 								 "--overwrite  : Overwrite previously displayed info with --watch and --compact instead of continuously logging\n"
-								 "--color      : Print headers in color in --compact mode for better readability\n";
+								 "--color      : Print headers in color in --compact mode for better readability\n"
+								 "--no-reasons : Do not query NVML for clock reasons (can cause stuttering)\n"
+								 "--board      : Also print temperatures from the CPU, motherboard, and other sensors\n";
 								 
-void print_gpu_info(int gpu_num, struct card_info gpus[], int compact);
+void print_gpu_info(int gpu_num, struct card_info gpus[]);
+void print_board_info(struct hwmon_avail_sensor *board_sensors, int num_sensors);
 
 int main (int argc, char **argv)
 {
 	struct card_info gpus[MAX_GPUS];
-	int gpu_count;
+	struct hwmon_avail_sensor board_sensors[MAX_BOARD_SENSORS];
+	int gpu_count, board_sensor_count;
 	int print_info = 0;			
-	int compact = 0;
+	int gpu_num = -1;				/* Card to control */
-	int gpu_num = -1;    /* Card to control */
+	int i2c_bus = -1;				/* Specific i2c bus to probe instead of all */
-	int overwrite = 0;
+	int overwrite = 0;				/* Overwrite printed console info in compact mode */		
-	unsigned int watch = 0;
+	unsigned int watch = 0;			/* Refresh display every this many seconds */
+	int print_board_sensors = 0;	/* Print CPU/motherbord/other sensors as well */
 	char *fan_speed[ICX3_MAX_FANS] = {NULL}; 
 	
 	/* Input parsing */
 	for (int i = 1; i < argc; i++){
-		if (strcmp(argv[i], "--gpu") == 0) {
+		if (strcmp(argv[i], "--i2c") == 0) {
+			i++;
+			if (i < argc) {
+				i2c_bus = atoi(argv[i]);
+			} else {
+				printf(helpstring);
+				return -1;
+			}
+		} else if (strcmp(argv[i], "--gpu") == 0) {
 			i++;
 			if (i < argc) {
 				gpu_num = atoi(argv[i]);
@@ -100,6 +123,10 @@ int main (int argc, char **argv)
 		} else if (strcmp(argv[i], "--color") == 0) {
 			header_start = HEADER_COLOR_START;
 			header_end = HEADER_COLOR_END;
+		} else if (strcmp(argv[i], "--no-reasons") == 0) {
+			no_reasons = 1;
+		} else if (strcmp(argv[i], "--board") == 0) {
+			print_board_sensors = 1;
 		} else {
 			printf(helpstring);
 			return 0;
@@ -119,21 +146,33 @@ int main (int argc, char **argv)
 	if (overwrite && !compact)
 		overwrite = 0;
 	
-	gpu_count = find_evga_gpu_i2cs(gpus, MAX_GPUS);
+	/* Scan for supported GPUs */
+	gpu_count = find_evga_gpu_i2cs(gpus, MAX_GPUS, i2c_bus);
+	/* Scan for supported Blackwell GPUs */
+	gpu_count = find_blackwell_gpus(gpus, gpu_count, MAX_GPUS);
 
+	/* Check for no GPUs found or other errors */
 	if (gpu_count == -1) {
 		printf("Error scanning I2C devices\n");
 		return -1;
 	} else if (gpu_count == 0) {
 		printf("No supported GPUs found.\nAre you root or do you have udev access to i2c devices?\nDo you need to run `modprobe i2c-dev`?\n");
-		return -1;
 	}
-	
+	/* Check for invalid GPUs */
 	if (gpu_num > gpu_count - 1) {
 		printf("Invalid GPU number specified (%d, max %d)\n", gpu_num, gpu_count - 1);
 		return -1;
 	}
 		
+	/* Scan for motherboard/CPU/other sensors */
+	if (print_board_sensors) {
+		board_sensor_count = find_board_sensors(board_sensors, MAX_BOARD_SENSORS);
+		if (init_rapl() && board_sensor_count < MAX_BOARD_SENSORS) {
+			board_sensors[board_sensor_count] = rapl_sensor;
+			zen3_rapl_sensor = board_sensor_count;
+			board_sensor_count++;
+		}
+	}
 	
 	/* execute fan commands */
 	if (gpu_num == -1) {
@@ -153,37 +192,61 @@ int main (int argc, char **argv)
 	/* NVML init */
 #ifdef USE_NVML
     init_nvml();
+	for (int i = 0; i < gpu_count; i++) {
+		gpus[i].nvml_device = malloc(sizeof(nvmlDevice_t));
+		get_nvml_handle(&gpus[i]);
+	}
 #endif
 
-	/* PCI init for VRAM/hotspot temps */
+#ifdef USE_DCGM
-#ifdef USE_LIBPCI
+	init_dcgm();
-	for (int i = 0; i < gpu_count; i++)
-		init_gddr6(&gpus[i]);
 #endif
 
 	/* print sensor info */
 	if (print_info) {
 		do {
-			printf("\x1b[K");	/* Clear current console line (really just for overwrite mode) */
+			if (overwrite)
+				printf("\x1b[K");	/* Clear current console line */
+			
+			if (print_board_sensors)
+				print_board_info(board_sensors, board_sensor_count);
 
 			if (gpu_num == -1) {
 				/* No GPU specified on command line, loop over all supported GPUs */
 				for (int i = 0; i < gpu_count; i++){
 					if (i > 0)
 						printf("\n");
-					print_gpu_info(i, &gpus[i], compact);
+					if (gpus[i].i2c_dev_path == NULL) {
+						print_blackwell_info(compact, overwrite, &gpus[i]);
+					} else {
+						print_gpu_info(i, &gpus[i]);
+					}
 				}		
 			} else if (gpu_num <= gpu_count - 1) {
-				print_gpu_info(gpu_num, &gpus[gpu_num], compact);
+				print_gpu_info(gpu_num, &gpus[gpu_num]);
 			}
 			
+#ifdef USE_DCGM
+			if (compact && (gpu_count > 0))
+				printf("\n");
+			print_dcgm(compact, overwrite);
+#endif
+
 			if (!overwrite)
 				printf("\n"); /* Print line break at the end for continuous output */
 			
 			if (overwrite && compact) {
+				int total_lines = 0;
+				if (print_board_sensors) total_lines++;
+				if (gpu_count > 0) {
+					total_lines += (gpu_num == -1) ? gpu_count : 1;
+				}
+#ifdef USE_DCGM
+				total_lines++;
+#endif
 				printf("\x1b[1G");						/* Move cursor back to column 1 */
-				if (gpu_count > 1)
+				if (total_lines > 1)
-					printf("\x1b[%dA", gpu_count-1);	/* Move cursor back up to the top of gpu list */
+					printf("\x1b[%dA", total_lines - 1);	/* Move cursor back up to the top */
 			}
 			
 			fflush(stdout);
@@ -194,9 +257,86 @@ int main (int argc, char **argv)
 #ifdef USE_NVML
 	nvmlShutdown();
 #endif
+#ifdef USE_DCGM
+	shutdown_dcgm();
+#endif
 }
 
-void print_gpu_info(int gpu_num, struct card_info *gpu, int compact) {
+void print_board_info(struct hwmon_avail_sensor *board_sensors, int num_sensors)
+{
+	int printed_sensors = 0;
+	int current_sort_index = 0;
+	float current_reading = 0.0;
+	int good_reading = 0;
+	/* These allow us to 'summarize' units and categories by only printing them when they change */
+	char *last_short_name = NULL;
+	char *last_units = NULL;
+	
+	float rapl_power;
+	
+	for (int i = 0; i < (sizeof(hwmon_sensor_info) / sizeof(struct hwmon_sensor)); i++) {
+		
+		/* Inject our Zen RAPL power reading here */
+		if (strcmp(hwmon_sensor_info[i].driver_name, "zen-rapl") == 0) {
+			board_sensors[zen3_rapl_sensor].sort_index = current_sort_index;
+			board_sensors[zen3_rapl_sensor].sensor_info = &hwmon_sensor_info[i];
+		}
+		
+		for (int j=0; j < num_sensors; j++) {
+			struct hwmon_avail_sensor *current_sensor = &board_sensors[j];
+
+			/* Loop over all sensors, but only output those with the current sort index so they come out sort of sorted
+			Duplicates (e.g.) multiple NVMe will come out in whatever sort of order the directory listing happened to */
+			if (current_sensor->sort_index == current_sort_index) {
+				printed_sensors++;
+				
+				if (j == zen3_rapl_sensor) {
+					good_reading = 1;
+					current_reading = get_rapl_package_power();
+				} else {
+					good_reading = get_sensor_reading(current_sensor, &current_reading);
+				}
+				
+				if (!good_reading)
+					continue;
+				
+				if (compact) {
+					/* Print units if needed */
+					if (last_units != NULL && strcmp(current_sensor->sensor_info->units, last_units))
+						printf("%s", last_units);
+					
+					/* Print new section header if needed */
+					if (last_short_name == NULL || strcmp(current_sensor->sensor_info->short_name, last_short_name)) {
+						if (last_short_name != NULL) /* Spacer for all headings not the first one */
+							printf(" ");
+						printf("%s%s%s", header_start, current_sensor->sensor_info->short_name, header_end);
+					}
+					printf(" %3.0f", current_reading);
+					
+					last_short_name = current_sensor->sensor_info->short_name;
+					last_units = current_sensor->sensor_info->units;
+					
+				} else {
+					printf("%s%s: %+.1f%s\n", 
+						current_sensor->sensor_info->name_prefix,
+						current_sensor->sensor_name,
+						current_reading,
+						current_sensor->sensor_info->units);
+				}
+				
+			}
+		}
+		current_sort_index++;
+	}
+	
+	if (compact && last_units != NULL)
+		printf("%s", last_units);
+	
+	printf("\n");
+}
+
+void print_gpu_info(int gpu_num, struct card_info *gpu)
+{
 	if (compact) {
 		/* One line per GPU */
 		printf("%s#%d FAN%s", header_start, gpu_num, header_end);
@@ -212,24 +352,22 @@ void print_gpu_info(int gpu_num, struct card_info *gpu, int compact) {
 		for (int i = 0; i < ICX3_NUM_TEMP_SENSORS; i++) {
 			if (i > 0 && strncmp(icx3_temp_sensor_names[i], icx3_temp_sensor_names[i-1], 3))
 				printf("%s %.3s%s", header_start, icx3_temp_sensor_names[i], header_end);
-#ifdef USE_LIBPCI
 			if (strncmp(icx3_temp_sensor_names[i], "MEM1", 4) == 0)
 				printf(" %3.0f", get_vram_temp(gpu));			/* Print the VRAM temp before the rest of the memory sensors */
-#endif
 			printf(" %3.0f", icx_temp_sensors[i]);
 		}
 		
-#ifdef USE_LIBPCI
 		printf("%s HOT%s %3.0f", header_start, header_end, get_hotspot_temp(gpu));
-#endif
 		
 		printf("°C ");
 		
 #ifdef USE_NVML
 		printf("%s MEM %s", header_start, header_end);
 		printf("%3d%%", get_nvml_mem_util(gpu));
-		printf("%s CLK %s", header_start, header_end);
+		if (!no_reasons) {
-		print_nvml_clock_reason(1, gpu);
+			printf("%s CLK %s", header_start, header_end);
+			print_nvml_clock_reason(1, gpu);
+		}
 #endif
 
 
@@ -244,26 +382,23 @@ void print_gpu_info(int gpu_num, struct card_info *gpu, int compact) {
 		float icx_temp_sensors[ICX3_NUM_TEMP_SENSORS] = {};
 		get_temp_sensors(icx_temp_sensors, gpu);
 		for (int i = 0; i < ICX3_NUM_TEMP_SENSORS; i++) {
-#ifdef USE_LIBPCI
 			if (strncmp(icx3_temp_sensor_names[i], "MEM1", 4) == 0)
 				printf("VRAM: +%.0f°C\n", get_vram_temp(gpu));			/* Print the VRAM temp before the rest of the memory sensors */
-#endif
 			printf("%s: %+.1f°C\n",
-			icx3_temp_sensor_names[i],
+				icx3_temp_sensor_names[i],
-			icx_temp_sensors[i]);
+				icx_temp_sensors[i]);
 		}
 		
-#ifdef USE_LIBPCI
 		printf("HotSpot: +%.0f°C\n", get_hotspot_temp(gpu));
-#endif
 
 #ifdef USE_NVML
 		printf("Mem util: %d%%\n", get_nvml_mem_util(gpu));
-		printf("Clock reasons: ");
+		if (!no_reasons) {
-		print_nvml_clock_reason(0, gpu);
+			printf("Clock reasons: ");
+			print_nvml_clock_reason(0, gpu);
+		}
 		printf("\n");
 #endif
 	}
-	
 }
 	

gddr6.c

@@ -1,105 +1,48 @@
 #include <string.h>
 #include <stdio.h>
-#include <pci/pci.h>
-#include <sys/mman.h>
-#include <fcntl.h>
-#include <unistd.h>
-
-#define PG_SZ sysconf(_SC_PAGE_SIZE)
 
 #include "gddr6.h"
 
-void init_gddr6(struct card_info *card)
+static struct device_offset device_offset_info[] = 
 {
-	/* Parse the address of the card to get the PCI info */
+	{.device_id = NVIDIA_RTX3090TI_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3090 Ti */
-	char pci_address[] = "00000000:00:00.0";
+	{.device_id = NVIDIA_RTX3090_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3090 */
-	int len = strlen(card->pci_id);
+	{.device_id = NVIDIA_RTX3080TI_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 Ti */
-	strcpy(&pci_address[sizeof(pci_address) - len - 1], card->pci_id);
+	{.device_id = NVIDIA_RTX3080_12G_LHR_DEV,	.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 12G LHR */
-	
+	{.device_id = NVIDIA_RTX3080_LHR_DEV,		.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 LHR */
-	int domain = 0;
+	{.device_id = NVIDIA_RTX3080_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 */
-	int bus = 0;
+	{.device_id = NVIDIA_RTX3070TI_GA102_DEV,	.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 Ti GA102 TODO:check */
-	int dev = 0;
+	{.device_id = NVIDIA_RTX3070TI_DEV,			.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 Ti TODO:check */
-	int func = 0;
+	{.device_id = NVIDIA_RTX3070_LHR_DEV,		.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 LHR */
-	
+	{.device_id = NVIDIA_RTX3070_DEV,			.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 */
-	sscanf(pci_address, "%x:%x:%x.%x", &domain, &bus, &dev, &func);
+};
-	
-	struct pci_access *pacc = NULL;
-    struct pci_dev *pci_dev = NULL;
-
-	pacc = pci_alloc();
-	pci_init(pacc);
-
-	pci_dev = pci_get_dev(pacc, domain, bus, dev, func);
-
-	pci_fill_info(pci_dev, PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS);
-
-	card->bar0 = (pci_dev->base_addr[0] & 0xFFFFFFFF);
-
-	pci_cleanup(pacc);
-}
 
 float get_vram_temp(struct card_info *card)
 {
-	int fd;
+	if (card->bar0_mmap == NULL)
-	float temp = 0.0;
+		return 0.0;
-	
-	if ((fd = open("/dev/mem", O_RDWR | O_SYNC)) == -1)
-    {
-        printf("Can't read memory. If you are root, enable kernel parameter iomem=relaxed\n");
-		return 0;
-    }
 
 	for (int i = 0; i < sizeof(device_offset_info) / sizeof(struct device_offset); i++) {
-		if (card->pci_device_id == device_offset_info[i].device_id){
+		if (card->pci_device_id == device_offset_info[i].device_id) {
-			unsigned int phys_addr = (card->bar0 + device_offset_info[i].vram_offset);
+			unsigned int read_result = *((unsigned int *)((char *)card->bar0_mmap + device_offset_info[i].vram_offset));
-			unsigned int base_offset = phys_addr & ~(PG_SZ-1);
+			return (float)((read_result & 0x00000fff) / 0x20);
-			void *map_base = mmap(0, PG_SZ, PROT_READ, MAP_SHARED, fd, base_offset);
-			if(map_base == (void *) -1)
-			{
-				if (fd != -1)
-					close(fd);
-				printf("Can't read memory for VRAM temperature. If you are root, enable kernel parameter iomem=relaxed\n");
-			}
-			void *virt_addr = (char *) map_base + (phys_addr - base_offset);
-			int read_result = *((unsigned int *) virt_addr);
-			temp = ((read_result & 0x00000fff) / 0x20);
-			munmap(map_base, PG_SZ);
 		}
 	}
 
-	close(fd);
+	return 0.0;
-	return temp;
 }
 
 float get_hotspot_temp(struct card_info *card)
 {
-	int fd;
+	if (card->bar0_mmap == NULL)
-	float temp = 0.0;
+		return 0.0;
-	
-	if ((fd = open("/dev/mem", O_RDWR | O_SYNC)) == -1)
-    {
-        printf("Can't read memory. If you are root, enable kernel parameter iomem=relaxed\n");
-		return 0;
-    }
 
 	for (int i = 0; i < sizeof(device_offset_info) / sizeof(struct device_offset); i++) {
-		if (card->pci_device_id == device_offset_info[i].device_id){
+		if (card->pci_device_id == device_offset_info[i].device_id) {
-			unsigned int phys_addr = (card->bar0 + device_offset_info[i].hotspot_offset);
+			unsigned int read_result = *((unsigned int *)((char *)card->bar0_mmap + device_offset_info[i].hotspot_offset));
-			unsigned int base_offset = phys_addr & ~(PG_SZ-1);
+			return (float)((read_result >> 8) & 0xff);
-			void *map_base = mmap(0, PG_SZ, PROT_READ, MAP_SHARED, fd, base_offset);
-			if(map_base == (void *) -1)
-			{
-				if (fd != -1)
-					close(fd);
-				printf("Can't read memory for hotspot. If you are root, enable kernel parameter iomem=relaxed\n");
-			}
-			void *virt_addr = (char *) map_base + (phys_addr - base_offset);
-			int read_result = *((unsigned int *) virt_addr);
-			temp = (read_result >> 8) & 0xff;
-			munmap(map_base, PG_SZ);
 		}
 	}
 
-	close(fd);
+	return 0.0;
-	return temp;
 }

gddr6.h

@@ -1,3 +1,5 @@
+#ifndef GDDR6_H
+#define GDDR6_H
 #include "evga-card.h"
 
 struct device_offset {
@@ -6,21 +8,6 @@ struct device_offset {
 	int hotspot_offset;
 };
 
-static struct device_offset device_offset_info[] = 
-{
-	{.device_id = NVIDIA_RTX3090TI_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3090 Ti */
-	{.device_id = NVIDIA_RTX3090_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3090 */
-	{.device_id = NVIDIA_RTX3080TI_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 Ti */
-	{.device_id = NVIDIA_RTX3080_12G_LHR_DEV,	.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 12G LHR */
-	{.device_id = NVIDIA_RTX3080_LHR_DEV,		.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 LHR */
-	{.device_id = NVIDIA_RTX3080_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 */
-	{.device_id = NVIDIA_RTX3080_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 */
-	{.device_id = NVIDIA_RTX3070TI_GA102_DEV,	.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 Ti GA102 TODO:check */
-	{.device_id = NVIDIA_RTX3070TI_DEV,			.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 Ti TODO:check */
-	{.device_id = NVIDIA_RTX3070_LHR_DEV,		.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 */
-	{.device_id = NVIDIA_RTX3070_DEV,			.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 LHR */
-};
-
-void init_gddr6(struct card_info *card);
 float get_vram_temp(struct card_info *card);
 float get_hotspot_temp(struct card_info *card);
+#endif

icx3.c

@@ -237,6 +237,6 @@ void set_fan(int fan, char *setting, struct card_info *card)
 	if (fan_readback.fanmode    != fan_control.fanmode ||
 		fan_readback.rpm_offset != fan_control.rpm_offset ||
 		fan_readback.duty       != fan_control.duty)
-		printf("Error setting fan %d on %s\n", fan, card->i2c_fd);
+		printf("Error setting fan %d on %s\n", fan, card->i2c_dev_path);
 }
 

icx3.h

@@ -1,6 +1,7 @@
 #ifndef ICX3_H
 #define ICX3_H
 
+#include <stdint.h>
 #include "evga-card.h"
 
 #define ICX3_I2C_ADDR 0x2D
@@ -22,26 +23,26 @@
 #define ICX3_WRITE_DISABLED 0xFE
 
 struct icx3_fan_control {
-	unsigned char length;
+	uint8_t length;
-	unsigned char fanmode;
+	uint8_t fanmode;
-	unsigned short rpm_offset;
+	uint16_t rpm_offset;
-	unsigned char duty;
+	uint8_t duty;
-	unsigned char duty_status;
+	uint8_t duty_status;
-	unsigned short rpm_status;
+	uint16_t rpm_status;
 };
 
 struct icx3_temp_sensors {
-	unsigned char length;
+	uint8_t length;
-	unsigned char data[18];
+	uint8_t data[18];
 };
 
 struct icx3_info {
-	unsigned char length;
+	uint8_t length;
-	unsigned char reserved;
+	uint8_t reserved;
-	unsigned char slave_address;
+	uint8_t slave_address;
-	unsigned char product_id;
+	uint8_t product_id;
-	unsigned char major_version;
+	uint8_t major_version;
-	unsigned char minor_version;
+	uint8_t minor_version;
 };
 
 enum icx3_product_id {

nvidia-dcgm.c

@@ -0,0 +1,127 @@
+#include <stdio.h>
+
+#include "nvidia-dcgm.h"
+
+/* Yeah I know this is duplicated */
+#define HEADER_COLOR_START "\x1b[36m"
+#define HEADER_COLOR_END "\x1b[39m"
+
+dcgmHandle_t 	dcgm_handle = (dcgmHandle_t)NULL;
+dcgmFieldGrp_t 	field_group_id;
+
+unsigned long long clock_reasons = 0;
+
+void init_dcgm()
+{
+	/* Init DCGM */
+	dcgmReturn_t result;
+	result = dcgmInit();
+	if (result != DCGM_ST_OK)
+		printf("Could not init DCGM: %s\n", errorString(result));
+	
+	/* Start embedded */
+	result = dcgmStartEmbedded(DCGM_OPERATION_MODE_MANUAL, &dcgm_handle);
+	if (result != DCGM_ST_OK)
+		printf("Could not start embedded DCGM engine: %s\n", errorString(result));
+	
+	/* Create metric group */
+	int num_metrics = sizeof(metric_readings) / sizeof(metric_readings[0]);
+	unsigned short field_ids[num_metrics + 1];
+	for (int i = 0; i < num_metrics; i++) {
+		field_ids[i] = metric_readings[i].field_id;
+		metric_readings[i].last_reading = 0.0;
+	}
+	field_ids[num_metrics] = DCGM_FI_DEV_CLOCKS_EVENT_REASONS;
+	result = dcgmFieldGroupCreate(dcgm_handle, num_metrics + 1, &field_ids[0], (char *)"evga_icx_fields", &field_group_id);
+	if (result != DCGM_ST_OK)
+		printf("Could not create DCGM metric group: %s\n", errorString(result));
+	
+	/* Start recording */
+	result = dcgmWatchFields(dcgm_handle, DCGM_GROUP_ALL_GPUS , field_group_id, 1000000, 30, 1);
+	if (result != DCGM_ST_OK)
+		printf("Could not record DCGM metrics: %s\n", errorString(result));
+}
+
+void shutdown_dcgm()
+{
+	dcgmStopEmbedded(dcgm_handle);
+	dcgmShutdown();
+}
+
+void print_dcgm(int compact, int overwrite)
+{
+	dcgmReturn_t result;
+	dcgmUpdateAllFields(dcgm_handle, 1);
+	result = dcgmGetLatestValues(dcgm_handle, DCGM_GROUP_ALL_GPUS, field_group_id, &update_field_values, NULL);
+	if (result != DCGM_ST_OK)
+		printf("Could not read DCGM metrics: %s\n", errorString(result));
+	
+	/* Print the float metrics */
+	for (int i = 0; i < sizeof(metric_readings) / sizeof(metric_readings[0]); i++) {
+		if (compact)
+			printf("%s%s%s%s %4.1f",
+				i==0 ? "": " ",
+				HEADER_COLOR_START,
+				metric_readings[i].short_name,
+				HEADER_COLOR_END,
+				metric_readings[i].last_reading * 100);
+		else
+			printf("%s: %.1f%\n", metric_readings[i].long_name, metric_readings[i].last_reading * 100);
+	}
+	if (compact)
+		printf("%%");
+	
+	/* Print clock reasons */
+	int single_reason = 1;
+	if (compact)
+		printf("%s CLK %s", HEADER_COLOR_START, HEADER_COLOR_END);
+	else
+		printf("Clock reasons: ");
+	
+	for (int i = 0; i < (sizeof(dcgm_clock_reason_names) / sizeof(struct clock_reason)); i++) {
+		if (clock_reasons & dcgm_clock_reason_names[i].mask) {
+			if (!single_reason) {
+				if (compact)
+					printf(",");
+				else
+					printf(", ");
+			}	
+			single_reason = 0;
+			
+			if (compact)
+				printf("%s", dcgm_clock_reason_names[i].short_name);
+			else
+				printf("%s", dcgm_clock_reason_names[i].long_name);
+		}
+	}
+	
+	if (single_reason)
+		printf("None");
+	
+	if (!compact)
+		printf(" (0x%llx)", clock_reasons);
+	else
+		printf("%-15s", " ");
+}
+
+int update_field_values(unsigned int gpu_id, dcgmFieldValue_v1 *values, int num_values, void *userdata)
+{
+	for (int j = 0; j < num_values; j++) {
+		if (values[j].fieldType == DCGM_FT_DOUBLE) {
+			/* A double reading so read it into our readings array */
+			for (int i = 0; i < sizeof(metric_readings) / sizeof(metric_readings[0]); i++) {
+				if (metric_readings[i].field_id == values[j].fieldId) {
+					double val = values[j].value.dbl;
+					if (val >= 1)
+						metric_readings[i].last_reading = 0.0;
+					else
+						metric_readings[i].last_reading = values[j].value.dbl;
+				}
+			}
+		} else if (values[j].fieldType == DCGM_FT_INT64) {
+			/* Int is probably the bitmask of our clock reasons */
+			clock_reasons = values[j].value.i64;
+		}
+	}
+	return 0;
+}

nvidia-dcgm.h

@@ -0,0 +1,45 @@
+#include <dcgm_agent.h>
+#include <dcgm_structs.h>
+
+#include "evga-card.h"
+
+struct metric_reading {
+	unsigned short field_id;
+	char *short_name;
+	char *long_name;
+	double last_reading;
+};
+
+static struct metric_reading metric_readings[] =
+{
+	{DCGM_FI_PROF_GR_ENGINE_ACTIVE,			"GFX",		"Graphics Engine Activity"},
+	{DCGM_FI_PROF_SM_ACTIVE,				"SM ACT",	"SM Activity"},
+	{DCGM_FI_PROF_SM_OCCUPANCY,				"OCC",		"SM Occupancy"},
+	{DCGM_FI_PROF_PIPE_FP16_ACTIVE,			"FP16",		"FP16 Engine Activity"},
+	{DCGM_FI_PROF_PIPE_FP32_ACTIVE,			"FP32",		"FP32 Engine Activity"},
+	{DCGM_FI_PROF_PIPE_FP64_ACTIVE,			"FP64",		"FP64 Engine Activity"},
+	{DCGM_FI_PROF_PIPE_INT_ACTIVE,			"INT",		"Integer Activity"},
+	{DCGM_FI_PROF_PIPE_TENSOR_ACTIVE,		"TENS",		"Tensor Activity"},
+	{DCGM_FI_PROF_PIPE_TENSOR_HMMA_ACTIVE,	"HMMA",		"Half Precision Tensor"},
+	{DCGM_FI_PROF_PIPE_TENSOR_IMMA_ACTIVE,	"IMMA",		"Integer Tensor"},
+	{DCGM_FI_PROF_PIPE_TENSOR_DFMA_ACTIVE,	"DFMA",		"Double Precision Tensor"},
+	{DCGM_FI_PROF_DRAM_ACTIVE,				"DRAM",		"Memory BW Utilization"}
+};
+
+static struct clock_reason dcgm_clock_reason_names[] = 
+{
+	{DCGM_CLOCKS_EVENT_REASON_GPU_IDLE,			"Idle", 	"GPU idle"},
+	{DCGM_CLOCKS_EVENT_REASON_CLOCKS_SETTING,	"AppClk",	"Application clocks"},
+	{DCGM_CLOCKS_EVENT_REASON_SW_POWER_CAP,		"Pwr",		"Power cap"},
+	{DCGM_CLOCKS_EVENT_REASON_HW_SLOWDOWN,		"HWSlow",	"Hardware slowdown"},
+	{DCGM_CLOCKS_EVENT_REASON_SYNC_BOOST,		"Sync",		"Sync boost"},
+	{DCGM_CLOCKS_EVENT_REASON_SW_THERMAL,		"SWTherm",	"Software thermal"},
+	{DCGM_CLOCKS_EVENT_REASON_HW_THERMAL,		"HWTherm",	"Hardware thermal"},
+	{DCGM_CLOCKS_EVENT_REASON_HW_POWER_BRAKE,	"HWPower",	"Hardware power brake"},
+	{DCGM_CLOCKS_EVENT_REASON_DISPLAY_CLOCKS,	"DispClk",	"Display clock"}
+};
+
+void init_dcgm();
+void shutdown_dcgm();
+void print_dcgm(int compact, int overwrite);
+int update_field_values(unsigned int gpu_id, dcgmFieldValue_v1 *values, int num_values, void *userdata);

nvidia-sensors.c

@@ -10,15 +10,14 @@ void init_nvml()
 		printf("Could not init NVML: %s\n", nvmlErrorString(result));
 }
 
-int get_nvml_handle(nvmlDevice_t *device, struct card_info *card)
+void get_nvml_handle(struct card_info *card)
 {
 	nvmlReturn_t result;
-	result = nvmlDeviceGetHandleByPciBusId_v2(card->pci_id, device);
+	result = nvmlDeviceGetHandleByPciBusId_v2(card->pci_id, card->nvml_device);
 	if (result != NVML_SUCCESS) {
-		printf("Failed to get device handle for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
+		printf("Failed to get NVML device handle for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
-		return 0;
+		card->nvml_device = NULL;
 	}
-	return 1;
 }
 
 void print_nvml_clock_reason(int compact, struct card_info *card)
@@ -26,8 +25,8 @@ void print_nvml_clock_reason(int compact, struct card_info *card)
 	unsigned long long reasons = get_nvml_clock_reasons(card);
 	int single_reason = 1;
 		
-	for (int i = 0; i < (sizeof(clock_reason_names) / sizeof(struct clock_reason)); i++) {
+	for (int i = 0; i < (sizeof(nvml_clock_reason_names) / sizeof(struct clock_reason)); i++) {
-		if (reasons & clock_reason_names[i].mask) {
+		if (reasons & nvml_clock_reason_names[i].mask) {
 			if (!single_reason) {
 				if (compact)
 					printf(",");
@@ -37,9 +36,9 @@ void print_nvml_clock_reason(int compact, struct card_info *card)
 			single_reason = 0;
 			
 			if (compact)
-				printf("%s", clock_reason_names[i].short_name);
+				printf("%s", nvml_clock_reason_names[i].short_name);
 			else
-				printf("%s", clock_reason_names[i].long_name);
+				printf("%s", nvml_clock_reason_names[i].long_name);
 		}
 	}
 	
@@ -48,31 +47,33 @@ void print_nvml_clock_reason(int compact, struct card_info *card)
 	
 	if (!compact)
 		printf(" (0x%llx)", reasons);
+	else
+		printf("%-15s", " ");
 }
 
 unsigned int get_nvml_temp(struct card_info *card)
 {
-	nvmlDevice_t nvml_device;
+	if (card->nvml_device == NULL)
-	if (!get_nvml_handle(&nvml_device, card))
 		return 0;
 	
-	unsigned int temp;
+	nvmlTemperature_t temp;
-	nvmlReturn_t result = nvmlDeviceGetTemperature(nvml_device, NVML_TEMPERATURE_GPU, &temp); 
+	temp.version = nvmlTemperature_v1;
+	temp.sensorType = NVML_TEMPERATURE_GPU;
+	nvmlReturn_t result = nvmlDeviceGetTemperatureV(*(nvmlDevice_t*)(card->nvml_device), &temp); 
 	if (result != NVML_SUCCESS) {
 		printf("Failed to get temperature for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
 		return 0;
 	}
-	return temp;
+	return temp.temperature;
 }
 
 unsigned long long get_nvml_clock_reasons(struct card_info *card)
 {
-	nvmlDevice_t nvml_device;
+	if (card->nvml_device == NULL)
-	if (!get_nvml_handle(&nvml_device, card))
 		return 0;
 	
 	unsigned long long reasons;
-	nvmlReturn_t result = nvmlDeviceGetCurrentClocksEventReasons(nvml_device, &reasons) ; 
+	nvmlReturn_t result = nvmlDeviceGetCurrentClocksEventReasons(*(nvmlDevice_t*)(card->nvml_device), &reasons) ; 
 	if (result != NVML_SUCCESS) {
 		printf("Failed to get clock reasons for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
 		return 0;
@@ -83,12 +84,11 @@ unsigned long long get_nvml_clock_reasons(struct card_info *card)
 
 unsigned int get_nvml_mem_util(struct card_info *card)
 {
-	nvmlDevice_t nvml_device;
+	if (card->nvml_device == NULL)
-	if (!get_nvml_handle(&nvml_device, card))
 		return 0;
 	
 	nvmlUtilization_t util;
-	nvmlReturn_t result = nvmlDeviceGetUtilizationRates(nvml_device, &util);
+	nvmlReturn_t result = nvmlDeviceGetUtilizationRates(*(nvmlDevice_t*)(card->nvml_device), &util);
 	if (result != NVML_SUCCESS) {
 		printf("Failed to get clock reasons for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
 		return 0;

nvidia-sensors.h

@@ -2,13 +2,7 @@
 
 #include "evga-card.h"
 
-struct clock_reason {
+static struct clock_reason nvml_clock_reason_names[] = 
-	unsigned long long mask;
-	char *short_name;
-	char *long_name;
-};
-
-static struct clock_reason clock_reason_names[] = 
 {
 	{nvmlClocksEventReasonGpuIdle,						"Idle", 	"GPU idle"},
 	{nvmlClocksEventReasonApplicationsClocksSetting,	"AppClk",	"Application clocks"},
@@ -22,7 +16,7 @@ static struct clock_reason clock_reason_names[] =
 };
 
 void init_nvml();
-int get_nvml_handle(nvmlDevice_t *device, struct card_info *card);
+void get_nvml_handle(struct card_info *card);
 void print_nvml_clock_reason(int compact, struct card_info *card);
 unsigned int get_nvml_temp(struct card_info *card);
 unsigned long long get_nvml_clock_reasons(struct card_info *card);

zen3-rapl.c

@@ -0,0 +1,112 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <math.h>
+#include <string.h>
+#include <fcntl.h>
+
+#include "zen3-rapl.h"
+
+#define AMD_STRING "AuthenticAMD"
+#define ZEN_FAMILY 0x17
+#define ZEN3_FAMILY 0x19
+
+#define MEASUREMENT_TIME 0.25
+
+static float energy_unit = 0;
+
+static int msr_file = -1;
+
+static float package_power = 0.0;
+
+static int check_zen()
+{
+    unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0, ext_family;
+    char vendor[13];
+
+    __get_cpuid(0, &eax, &ebx, &ecx, &edx);
+
+    memcpy(vendor, &ebx, 4);
+    memcpy(vendor+4, &edx, 4);
+    memcpy(vendor+8, &ecx, 4);
+    vendor[12] = 0;
+
+    if (strcmp(vendor, AMD_STRING) != 0){
+        return 0;
+    }
+
+    __get_cpuid(1, &eax, &ebx, &ecx, &edx);
+
+    ext_family = ((eax >> 8) & 0xF) + ((eax >> 20) & 0xFF);
+    if (ext_family != ZEN_FAMILY && ext_family != ZEN3_FAMILY){
+        return 0;
+    }
+
+    return 1;
+}
+
+static int read_msr(int file, unsigned int index, unsigned long long *data)
+{
+    if (file < 0)
+        return 0;
+
+    return pread(file, data, sizeof *data, index) == sizeof *data;
+}
+
+static float get_energy_unit()
+{
+    unsigned long long data;
+    // AMD OSRR: page 139 - MSRC001_0299
+    if (!read_msr(msr_file, 0xC0010299, &data))
+        return 0.0;
+
+    return pow(1.0/2.0, (float)((data >> 8) & 0x1F));
+}
+
+static unsigned long get_package_energy()
+{
+    unsigned long long data;
+    // AMD OSRR: page 139 - MSRC001_029B
+    if (!read_msr(msr_file, 0xC001029B, &data))
+        return 0;
+
+    return data;
+}
+
+int init_rapl()
+{
+	/* Check for supported Zen CPU */
+	if (!check_zen())
+		return 0;
+	
+	/* Open MSR file */
+	char *msr_path = "/dev/cpu/0/msr";
+    msr_file = open(msr_path, O_RDONLY);
+	if (msr_file < 0)
+		return 0;
+	
+	/* Energy unit */
+	energy_unit = get_energy_unit();
+    if (energy_unit == 0)
+        return 0;
+	
+	return 1;
+}
+
+float get_rapl_package_power()
+{
+	unsigned long package_eng_b = 0;
+	unsigned long package_eng_a = 0;
+
+    package_eng_b = get_package_energy();
+
+    usleep(MEASUREMENT_TIME*1000000);
+
+    package_eng_a = get_package_energy();
+
+	/* Only update if we computed a good power reading, otherwise return the most recent measurement */
+    if (package_eng_a >= package_eng_b)
+        package_power = (package_eng_a - package_eng_b) * energy_unit / MEASUREMENT_TIME;
+	
+	return package_power;
+}

zen3-rapl.h

@@ -0,0 +1,6 @@
+#include "board-sensors.h"
+
+static struct hwmon_avail_sensor rapl_sensor = {"Package power", -1, NULL, NULL};
+
+int init_rapl();
+float get_rapl_package_power();