Fixes when compiling with both NVML and DCGM.

NVML temp code updated (driver version ~555 and up)

LICENSE

@@ -0,0 +1,8 @@
+MIT No Attribution
+
+Copyright 2025 admin@long-cat.net
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

Makefile

@@ -0,0 +1,29 @@
+.PHONY : clean debug
+
+OBJS = blackwell-sensors.o card.o evga-icx.o evga-card.o gddr6.o icx3.o board-sensors.o zen3-rapl.o
+LDLIBS = -li2c -lm
+CFLAGS = -MD -I/usr/local/cuda/targets/x86_64-linux/include/
+
+ifdef USE_NVML
+LDLIBS += -lnvidia-ml -L/usr/local/cuda/targets/x86_64-linux/lib/stubs/
+CFLAGS += -DUSE_NVML
+OBJS += nvidia-sensors.o
+endif
+
+ifdef USE_DCGM
+LDLIBS += -ldcgm
+CFLAGS += -DUSE_DCGM -I/usr/include/datacenter-gpu-manager-4/
+OBJS += nvidia-dcgm.o
+endif
+
+evga-icx : $(OBJS)
+
+debug : CFLAGS += -g -Og
+debug : evga-icx
+
+clean :
+	rm *.o
+	rm *.d
+	rm evga-icx
+	
+-include $(OBJS:.o=.d)

README.md

@@ -2,8 +2,10 @@
 
 This program allows you to read temperature sensors off of supported EVGA 30-series iCX3 video cards, as well as control the fans individually.
 
+Also a mashup of a bunch of other temperature sensor stuff too like Blackwell cards, lm_sensors, and DCGM metrics that I have just thrown in here.
+
 ## Prerequisites
-A supported EVGA 30-series iCX3 card.  I have not done extensive testing but belive this is every model of their:
+A supported EVGA 30-series card with iCX3.  This includes:
 * RTX 3060 Ti
 * RTX 3070
 * RTX 3070 Ti
@@ -14,21 +16,40 @@ A supported EVGA 30-series iCX3 card.  I have not done extensive testing but bel
 
 The number of fans supported depends, of course, on your particular model.
 
+You must have the `i2c-dev` kernel module loaded with `modprobe i2c-dev`
+
 Access to the `/dev/i2c` device files, which means either:
 * Run as root, or
 * Install udev rules to allow user access. If you have the OpenRGB udev rules installed to control the LEDs you already have this set up.
 
 ## Dependencies
 * libi2c-dev
+* libnvidia-ml-dev (if building with `USE_NVML=1`)
+* datacenter-gpu-manager-4-dev (if building with `USE_DCGM=1`)
 
 ## Building
 `make`
 
+## Optional features
+
+### NVML support
+Add the make flag `USE_NVML=1` and the it will also display the main GPU temperature ("GPU1") as reported by the NVIDIA driver.  It will also display the performance cap/clock reason and memory controller utilization.  This requires the NVIDIA management library (NVML) to be installed.
+
+### VRAM and Hotspot temperature
+Run as root and you can also read the VRAM and "hotspot" temperatures.  These require access the BAR0 through the `/sys/bus/pci/devices/*/resource0` device files.  These sensors are **extremely** undocumented so I can't say anything about their accuracy. If you have a Blackwell (50 series) card you can read all 4 die temp sensors, a 'system' temperature, and every individual VRAM chip temp (add your card in blackwell-temps.c).
+
+### Datacenter GPU Manager metrics
+Add the make flag `USE_DCGM=1` and it can display real-time metrics from supported Nvidia GPUs like SM occupancy, integer/fp pipe usage, tensor usage, and DRAM bandwidth utilization. This requires a Quadro/Workstation or a Tesla/Datacenter card, it does not work on GeForce cards.
+
+### Hardware monitoring
+This program can also monitor hardware sensors using the standard linux `hwmon` API.  See board-sensors.h for examples of how to configure these, they must be staticlly compiled in.
+
 ## Usage
 Note that when controlling fans directly through iCX3 they will fall offline from the Nvidia driver and show as 0 RPM until you return them to automatic mode.
 
 ```text
 Available options:
+--i2c N      : Only probe I2C bus N instead of all (may help with stuttering or freezing when probing I2C devices)
 --gpu N      : Control only GPU N instead of all supported cards
 --fan SPEED  : Set all fans at once to SPEED (see below)
 --fanN SPEED : Set fan N (0-3) to SPEED
@@ -38,6 +59,12 @@ Available options:
       [+/-]N to set that fan to an RPM offset from the GPU-controlled speed
 --reset      : Reset all fans to their default mode
 --sensors    : Print sensor readings even if setting a fan speed
+--compact    : Print sensor reading in a compact one-line per card format
+--watch N    : Keep printing output every N seconds
+--overwrite  : Overwrite previously displayed info with --watch and --compact instead of continuously logging new lines
+--color      : Print headers in color in --compact mode for better readability
+--no-reasons : Do not query NVML for clock reasons (can cause stuttering)
+--board      : Also print temperatures from the CPU, motherboard, and other sensors
 ```
 
 ### Examples:
@@ -45,19 +72,30 @@ Read sensors:
 ```text
 $ ./evga-icx
 #0: EVGA GeForce RTX 3090 FTW3 Ultra v2 (/dev/i2c-3) @ c:00.0
-Fan 0: 1094 RPM (36/0%, Auto)
-Fan 1: 1164 RPM (38/0%, Auto)
-Fan 2: 1161 RPM (38/0%, Offset)
+Fan 0: 1751 RPM (58/0%, Auto)
+Fan 1: 1730 RPM (57/0%, Auto)
+Fan 2: 1712 RPM (57/0%, Offset)
 Ext. fan: 0 RPM (0/0%, Offset)
-GPU2: 35.8 C
-MEM1: 35.3 C
-MEM2: 35.6 C
-MEM3: 35.9 C
-PWR1: 35.0 C
-PWR2: 35.6 C
-PWR3: 36.3 C
-PWR4: 36.9 C
-PWR5: 36.3 C
+GPU1: +65°C
+GPU2: +57.8°C
+VRAM: +74°C
+MEM1: +56.1°C
+MEM2: +53.5°C
+MEM3: +55.5°C
+PWR1: +48.2°C
+PWR2: +53.2°C
+PWR3: +59.6°C
+PWR4: +58.0°C
+PWR5: +51.1°C
+HotSpot: +75°C
+Mem util: 43%
+Clock reasons: Power cap (0x4)
+```
+
+Compact one-line mode:
+```text
+$ ./evga-icx --compact
+#0 FAN  59  59  58   0% GPU  66  60 MEM  74  58  55  58 PWR  49  55  61  60  53 HOT  77°C  MEM  42% CLK Pwr
 ```
 
 Set external fan to follow Nvidia driver controlled speed with a -500 RPM offset:

blackwell-sensors.c

@@ -0,0 +1,187 @@
+#include <stdio.h>
+#include <string.h>
+#include <dirent.h>
+#include <stdint.h>
+#include <math.h>
+
+#include "blackwell-sensors.h"
+
+/* Number of die sensors, 4 for Blackwell */
+#define BLACKWELL_NUM_DIE_SENSORS 4
+/* Memory spacing between DRAM FBPA blocks */
+#define BLACKWELL_FBPA_STRIDE 0x4000
+/* Maximum count of FBPA blocks, technically up to 18 but no consumer card has more than 8 */
+#define BLACKWELL_MAX_NUM_FBPA 9
+
+/* Yeah I know this is duplicated */
+#define HEADER_COLOR_START "\x1b[36m"
+#define HEADER_COLOR_END "\x1b[39m"
+
+static float read_temp_sensor(void *addr);
+static int read_dram_temp_sensor(void *addr);
+
+struct blackwell_card_info {
+	char *card_name;
+	unsigned short vendor_id;
+	unsigned short device_id;
+	int die_temp_offset[BLACKWELL_NUM_DIE_SENSORS];
+	int sys_temp_offset;
+	int dram_temp_offset_start;
+	int clamshell;			/* If the card has dual-side ram, if not then half the DRAM temps are repeated */
+};
+
+static struct blackwell_card_info blackwell_card_list[] =
+{
+	{"NVIDIA RTX PRO 6000 Blackwell Workstation Edition",	NVIDIA_VEN,		NVIDIA_RTXPRO6000_WORKSTATION_DEV,	{0xAD0A90, 0xAD0A94, 0xAD0A98, 0xAD0A9C},	0xAD06EC,	0x9024C0,	true}
+};
+
+
+/* Search all i2c device files for ones are on a PCI device of a supported GPU,
+   and respond with the correct iCX3 version information */
+int find_blackwell_gpus(struct card_info *infos, int num_gpus, int max_gpus)
+{
+	char pci_devices_path[PATH_MAX];
+	char device_path[PATH_MAX];
+	char dev_file[PATH_MAX];
+	char pci_path[PATH_MAX];
+	char *pci_addr;
+
+	DIR *dir;
+	struct dirent *ent;
+	
+	unsigned short pci_vendor, pci_device = 0;
+	
+	/* Start looking for PCI */
+	strcpy(pci_devices_path, "/sys/bus/pci/devices/");
+	dir = opendir(pci_devices_path);
+	
+	/* make sure we can open the PCI device */
+	if(dir == NULL)
+		return num_gpus;
+	
+	/* loop over all PCI devices */
+	while((ent = readdir(dir)) != NULL)
+	{				
+		strcpy(device_path, pci_devices_path);
+		strcat(device_path, ent->d_name);
+						
+		pci_vendor = read_pci_id(device_path, "/vendor");
+		pci_device = read_pci_id(device_path, "/device");
+		
+		/* See if it's a supported Blackwell card*/
+		for (int i = 0; i < (sizeof(blackwell_card_list) / sizeof(struct blackwell_card_info)); i++) {
+			
+			if (pci_vendor == blackwell_card_list[i].vendor_id &&
+			    pci_device == blackwell_card_list[i].device_id) {
+				
+				infos[num_gpus].card_name     = blackwell_card_list[i].card_name;
+				infos[num_gpus].pci_id        = strdup(ent->d_name);
+				infos[num_gpus].pci_dev_path  = strdup(device_path);
+				infos[num_gpus].pci_device_id = pci_device;
+				infos[num_gpus].i2c_dev_path  = NULL;
+				infos[num_gpus].i2c_fd        = -1;
+				
+				mmap_bar0(&infos[num_gpus]);
+				num_gpus++;
+				break;
+			}
+		}
+		
+		if (num_gpus == max_gpus)
+			break;
+	}
+	
+	return num_gpus;
+}
+
+void print_blackwell_info(int compact, int overwrite, struct card_info *gpu)
+{
+	if (gpu->bar0_mmap == NULL)
+		return;
+	
+	for (int i = 0; i < (sizeof(blackwell_card_list) / sizeof(struct blackwell_card_info)); i++) {
+		if (!compact)
+			printf("%s @ %s\n", gpu->card_name, gpu->pci_id);
+		float temp;
+		
+		if (blackwell_card_list[i].device_id == gpu->pci_device_id) {
+			/* Print die sensors */
+			if (compact)
+				printf("%sDIE%s", HEADER_COLOR_START, HEADER_COLOR_END);
+			for (int die_sens_num = 0; die_sens_num < BLACKWELL_NUM_DIE_SENSORS; die_sens_num++) {
+				temp = read_temp_sensor(gpu->bar0_mmap + blackwell_card_list[i].die_temp_offset[die_sens_num]);
+				if (compact)
+					printf(" %3.0f", temp);
+				else
+					printf("Die %d: %+.1f°C\n", die_sens_num, temp);
+			}
+			
+			/* Sys temp */
+			temp = read_temp_sensor(gpu->bar0_mmap + blackwell_card_list[i].sys_temp_offset);
+			if (compact)
+				printf(" %sSYS%s %3.0f", HEADER_COLOR_START, temp, HEADER_COLOR_END);
+			else
+				printf("System: %+.1f°C\n", temp);
+			
+			/* DRAM temps */
+			int dram_num = 0;
+			int num_channels = blackwell_card_list[i].clamshell ? 4 : 2;
+			int dram_max = 0;
+			int dram_min = 120;
+			int dram_total = 0.0;
+			for (int fbpa_num = 0; fbpa_num < BLACKWELL_MAX_NUM_FBPA; fbpa_num++) {
+				for (int chan_num = 0; chan_num < num_channels; chan_num++) {
+					temp = read_dram_temp_sensor(gpu->bar0_mmap + 
+						blackwell_card_list[i].dram_temp_offset_start + 
+						BLACKWELL_FBPA_STRIDE * fbpa_num +
+						0x4 * chan_num + 
+						2);
+						
+					if (temp == 0.0)
+						break;
+					
+					dram_max = (temp > dram_max) ? temp : dram_max;
+					dram_min = (temp < dram_min) ? temp : dram_min;
+					dram_total += temp;
+					
+					if (!compact)
+						printf("DRAM %i: %+.0f°C\n", dram_num, temp);
+					dram_num++;
+				}
+			}
+			if (compact) {
+				printf(" %sDRAM HI%s %3.0f %sAVG%s %5.1f %sLO%s %3.0f",
+					HEADER_COLOR_START,
+					HEADER_COLOR_END,
+					(float)dram_max,
+					HEADER_COLOR_START,
+					HEADER_COLOR_END,
+					(float)dram_total/dram_num,
+					HEADER_COLOR_START,
+					HEADER_COLOR_END,
+					(float)dram_min);
+			} else {
+				printf("DRAM average: %+.1f°C\n", (float)dram_total/dram_num);
+				printf("DRAM min: %+.0f°C\n", (float)dram_min);
+				printf("DRAM max: %+.0f°C\n", (float)dram_max);
+			}
+			return;
+		}
+	}
+}
+
+float read_temp_sensor(void *addr)
+{
+	return (*(uint16_t *)addr) / (float)0x100;
+}
+
+int read_dram_temp_sensor(void *addr)
+{
+	uint16_t raw = *(uint16_t *)addr;
+	if (raw == 0xBADF)
+		return 0.0;
+	
+	int code = (raw >> 8) & 0xFF;
+	if (code > 80) code = 80;
+	return (code > 19) ? (code - 20) * 2 : -(40 - code * 2);
+}

blackwell-sensors.h

@@ -0,0 +1,10 @@
+#ifndef BLACKWELL_SENSORS_H
+#define BLACKWELL_SENSORS_H
+
+#include "card.h"
+
+
+
+int find_blackwell_gpus(struct card_info *infos, int num_gpus, int max_gpus);
+void print_blackwell_info(int compact, int overwrite, struct card_info *gpu);
+#endif

board-sensors.c

@@ -0,0 +1,122 @@
+#include <dirent.h>
+#include <string.h>
+#include <stdlib.h>
+
+#include "board-sensors.h"
+
+int find_board_sensors(struct hwmon_avail_sensor *board_sensors, int max_sensors)
+{
+	const char *hwmon_path = "/sys/class/hwmon/";
+	char device_path[NAME_MAX];
+	char sensor_path[NAME_MAX];
+	
+	char driver_name[256];
+	
+	FILE *file;
+	DIR *dir;
+	struct dirent *ent;
+	
+	int num_sensors = 0;
+	
+	/* Start looking for hwmon devices in /sys/class/hwmon/ */
+	dir = opendir(hwmon_path);
+	
+	/* make sure we can open the device directory */
+	if(dir == NULL)
+		return 0;
+	
+	/* loop over all hwmon devices */
+	while((ent = readdir(dir)) != NULL)
+	{
+		/* Ignore any non-hwmon dirs */
+		if(strncmp(ent->d_name, "hwmon", 5) != 0) 
+			continue;
+		
+		strcpy(device_path, hwmon_path);
+		strcat(device_path, ent->d_name);
+		
+		/* Read in the name of the device */
+		strcpy(sensor_path, device_path);
+		strcat(sensor_path, "/name");
+		
+		file = fopen(sensor_path, "r");
+		if (file == NULL)
+			continue;
+		
+		if (fgets(driver_name, sizeof(driver_name), file) == NULL) {
+			fclose(file);
+			continue;
+		}
+		
+		fclose(file);
+
+		/* Driver names have a linebreak at the end so let's remove that for comparison*/
+		driver_name[strlen(driver_name) - 1] = '\0';
+		
+		/* Loop through all supported sensors and see if any are present in this device */
+		for (int i = 0; i < (sizeof(hwmon_sensor_info) / sizeof(struct hwmon_sensor)); i++) {
+			if (strcmp(driver_name, hwmon_sensor_info[i].driver_name) == 0) {
+				/* We matched the driver name, try to open the files */
+				strcpy(sensor_path, device_path);
+				strcat(sensor_path, "/");
+				strcat(sensor_path, hwmon_sensor_info[i].sensor_file_name);
+				strcat(sensor_path, "_input");
+				
+				file = fopen(sensor_path, "r");
+				if (file != NULL) {
+					fclose(file);
+					/* Good open of the sensor file */
+					board_sensors[num_sensors].file = calloc(NAME_MAX, sizeof(char));
+					strcpy(board_sensors[num_sensors].file, sensor_path);
+					board_sensors[num_sensors].sort_index = i;
+					board_sensors[num_sensors].sensor_info = &hwmon_sensor_info[i];
+					
+					/* Read in the sensor name */
+					board_sensors[num_sensors].sensor_name = calloc(MAX_SENSOR_NAME_LENGTH, sizeof(char));
+					strcpy(sensor_path, device_path);
+					strcat(sensor_path, "/");
+					strcat(sensor_path, hwmon_sensor_info[i].sensor_file_name);
+					strcat(sensor_path, "_label");
+					
+					file = fopen(sensor_path, "r");
+					if (file != NULL)
+						fgets(board_sensors[num_sensors].sensor_name, MAX_SENSOR_NAME_LENGTH, file);
+
+					/* Sensor name seems to always have a trailing newline we don't want */
+					size_t len_without_newline = strcspn(board_sensors[num_sensors].sensor_name, "\n");
+					board_sensors[num_sensors].sensor_name[len_without_newline] = '\0';
+					
+					if (num_sensors == max_sensors)
+						return num_sensors;
+					num_sensors++;
+				}
+			}
+		}
+		
+	}
+	
+	return num_sensors;
+}
+
+/* Returns 0 on a bad read or missing sensor, 1 on OK */
+int get_sensor_reading(struct hwmon_avail_sensor *sensor, float *reading) {
+	char buf[256] = {0};
+	long int raw;
+	FILE *file;
+	
+	file = fopen(sensor->file, "r");
+	if (file == NULL)
+		return 0;
+
+	fgets(buf, 256, file);
+	raw = strtol(buf, NULL, 10);
+	
+	fclose(file);
+	
+	*reading = (float)raw / sensor->sensor_info->divisor;
+
+	if (*reading == sensor->sensor_info->bad_value)
+		return 0;
+	
+	return 1;
+}

board-sensors.h

@@ -0,0 +1,61 @@
+#ifndef BOARD_SENSORS_H
+#define BOARD_SENSORS_H
+
+#include <stdio.h>
+
+#define MAX_SENSOR_NAME_LENGTH 256
+
+struct hwmon_sensor {
+	char *driver_name;				/* Contents of /sys/class/hwmon/hwmonX/name */
+	char *sensor_file_name;			/* Sysfs file to read */
+	char *name_prefix;				/* Prefix to attach to temp*_label for clarity */
+	char *short_name;				/* 'Category' name when using compact mode */
+	char *units;					/* Units string */
+	float divisor;					/* Divisor to convert temp* to units */
+	float bad_value;	/* Raw value that indicates a bad (missing) sensor TODO: verfiy most of these*/
+};
+
+/* Note the order here matters, it's the order these will be printed in */
+static struct hwmon_sensor hwmon_sensor_info[] = 
+{
+	{"zenpower",		"temp1",	"CPU ",			"CPU",	"°C",	1000.0,	-40.0	},	/* Tdie */
+	{"asusec",			"temp2",	"Motherboard ",	"CPU",	"°C",	1000.0,	-40.0	},	/* CPU */
+	{"zenpower",		"temp3",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd1 */
+	{"zenpower",		"temp4",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd2 */
+	{"zenpower",		"temp5",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd3 */
+	{"zenpower",		"temp6",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd4 */
+	{"zenpower",		"temp7",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd5 */
+	{"zenpower",		"temp8",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd6 */
+	{"zenpower",		"temp9",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd7 */
+	{"zenpower",		"temp10",	"CPU ",			"CCD",	"°C",	1000.0,	-40.0	},	/* Tccd8 */
+	{"zen-rapl",		"",			"CPU ",			"POW",	" W",	0.0,	0.0		},	/* Zen RAPL placeholder */
+	{"asusec",			"temp1",	"Motherboard ",	"CHIP",	"°C",	1000.0,	-40.0	},	/* Chipset */
+	{"asusec",			"temp5",	"Motherboard ",	"VRM",	"°C",	1000.0,	-40.0	},	/* VRM */
+	{"asusec",			"temp3",	"",				"MOBO",	"°C",	1000.0,	-40.0	},	/* Motherboard */
+	/*{"r8169_0_500:00",	"temp1",	"Ethernet",		"ETH",	"°C",	1000.0,	-40.0	},	/* Ethernet adapter */
+	{"nvme",			"temp1",	"NVMe ",		"NVME",	"°C",	1000.0,	-40.0	},	/* NVME Composite */
+	{"asusec",			"temp6",	"Motherboard ",	"AIR",	"°C",	1000.0,	-40.0	},	/* Water_In */
+	{"asusec",			"temp4",	"Motherboard ",	"AIR",	"°C",	1000.0,	-40.0	},	/* T_Sensor */
+	{"asusec",			"temp7",	"Motherboard ",	"AIR",	"°C",	1000.0,	-40.0	},	/* Water_Out */
+	{"nct6798",			"fan2",		"CPU fan",		"CPU",	"%",	15.0,	0.0		},	/* cpu_fan, cpu mid */
+	{"asusec",			"fan1",		"",				"CPU",	"%",	15.0,	0.0		},	/* cpu_opt, cpu front */
+	{"nct6798",			"fan5",		"H amp",		"CHA",	"%",	12.0,	0.0		},	/* h_amp, front fan */
+	{"nct6798",			"fan3",		"Chassis 2",	"CHA",	"%",	12.0,	0.0		},	/* cha2, top front */
+	{"nct6798",			"fan1",		"Chassis 1",	"CHA",	"%",	12.0,	0.0		},	/* cha1, top rear */
+	{"nct6798",			"fan4",		"Chassis 3",	"CHA",	"%",	13.0,	0.0		},	/* cha3, rear */
+	{"nct6798",			"fan6",		"AIO pump",		"CHA",	"%",	12.0,	0.0		},	/* aio_pump? */
+	{"nct6798",			"fan7",		"W pump",		"CHA",	"%",	12.0,	0.0		},	/* w_pump+? */
+	{"asusec",			"fan2",		"",				"CHIP",	"%",	35.0,	0.0		},	/* chipset */
+};
+
+struct hwmon_avail_sensor {
+	char *sensor_name;					/* Sensor name as read from the sysfs file */
+	int sort_index;						/* Sort index for order to display in */
+	char *file;							/* File to read from */
+	struct hwmon_sensor *sensor_info;	/* Associated sensor info struct */
+};
+
+int find_board_sensors(struct hwmon_avail_sensor *board_sensors, int max_sensors);
+int get_sensor_reading(struct hwmon_avail_sensor *sensor, float *reading);
+
+#endif

card.c

@@ -0,0 +1,96 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <limits.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+#include "card.h"
+
+/* Read a 16-bit unsigned int from a pci id file */
+unsigned short read_pci_id(char *device_path, char *field)
+{
+	char buf[16];
+	char file_path[PATH_MAX];
+	
+	strcpy(file_path, device_path);
+	strcat(file_path, field);
+	
+	FILE *fp = fopen(file_path, "r");
+	
+	if (fp == NULL)
+		return 0;
+		
+	if (fgets(buf, sizeof(buf), fp) == NULL) {
+		fclose(fp);
+		return 0;
+	}
+	
+	fclose(fp);
+	
+	return (unsigned short)strtoul(buf, NULL, 16);
+}
+
+/* Read a PCI ID from the name of an Nvidia i2c device */
+char *read_nvidia_pci_address(char *device_path)
+{
+	char file_path[PATH_MAX];
+	
+	char *ret = calloc(16 + 1, sizeof(char)); /* assuming pci ids could look as large as 00000000:0C:00.0 */
+	
+	strcpy(file_path, device_path);
+	strcat(file_path, "/name");
+	
+	FILE *fp = fopen(file_path, "r");
+	
+	if (fp == NULL) {
+		free(ret);
+		return NULL;
+	}
+	
+	fscanf(fp, "NVIDIA i2c adapter %*u at %16s", ret);
+	fclose(fp);
+	
+	if (strlen(ret) == 0) {
+		free(ret);
+		return NULL;
+	}
+	
+	return ret;
+}
+
+/* Open an mmap to the BAR0 of a PCI device using its /sys/bus/pci device files */
+void mmap_bar0(struct card_info *card)
+{
+	char resource_path[PATH_MAX];
+	snprintf(resource_path, sizeof(resource_path), "%s/resource0", card->pci_dev_path);
+	
+	/* Open FD on the resource file */
+	int fd = open(resource_path, O_RDONLY);
+	if (fd == -1) {
+		printf("Unable to open %s\n", resource_path);
+		card->bar0_mmap = NULL;
+		return;
+	}
+
+	/* Figure out the size of BAR0 fomr the size of the resource file */
+	struct stat st;
+	if (fstat(fd, &st) == -1) {
+		printf("Unable to stat %s\n", resource_path);
+		close(fd);
+		card->bar0_mmap = NULL;
+		return;
+	}
+
+	/* mmap the bar0 resource file */
+	card->bar0_mmap = mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
+	if (card->bar0_mmap == MAP_FAILED) {
+		printf("Unable to mmap %s\n", resource_path);
+		card->bar0_mmap = NULL;
+	}
+
+	/* mmap stays open after the FD has been closed */
+	close(fd);
+}

card.h

@@ -0,0 +1,48 @@
+#ifndef CARD_H
+#define CARD_H
+
+#define NVIDIA_VEN                                      0x10DE
+
+#define NVIDIA_RTX3060_DEV                              0x2503
+#define NVIDIA_RTX3060_GA104_DEV                        0x2487
+#define NVIDIA_RTX3060_GA106_DEV                        0x2503
+#define NVIDIA_RTX3060_LHR_DEV                          0x2504
+#define NVIDIA_RTX3060TI_DEV                            0x2486
+#define NVIDIA_RTX3060TI_GDDR6X_DEV                     0x24C9
+#define NVIDIA_RTX3060TI_LHR_DEV                        0x2489
+#define NVIDIA_RTX3060TI_V1_LHR_DEV                     0x2414
+#define NVIDIA_RTX3070_DEV                              0x2484
+#define NVIDIA_RTX3070_LHR_DEV                          0x2488
+#define NVIDIA_RTX3070TI_DEV                            0x2482
+#define NVIDIA_RTX3070TI_GA102_DEV                      0x2207
+#define NVIDIA_RTX3080_DEV                              0x2206
+#define NVIDIA_RTX3080_LHR_DEV                          0x2216
+#define NVIDIA_RTX3080_12G_LHR_DEV                      0x220A
+#define NVIDIA_RTX3080TI_DEV                            0x2208
+#define NVIDIA_RTX3090_DEV                              0x2204
+#define NVIDIA_RTX3090TI_DEV                            0x2203
+
+#define NVIDIA_RTXPRO6000_WORKSTATION_DEV				0x2BB1
+
+struct card_info {
+	char *card_name;					/* The 'nice' name of the card */
+	char *pci_id;						/* PCI bus address in domain:bus:device.function format. May be shortened (e.g. c:00.0) */
+	char *pci_dev_path;                 /* Path to the PCI device file */
+	void *bar0_mmap;                    /* Mmap of the PCI BAR0 */
+	unsigned short pci_device_id;		/* The device ID of the card, i.e. corresponds to the NVIDIA model number */
+	char *i2c_dev_path;					/* Path to the i2c device file */
+	int i2c_fd;							/* File descriptor for the i2c device file, for re-use */
+	int product_id;						/* EVGA internal product ID, as reported by the iCX3 controller */
+	void *nvml_device;					/* Pointer to nvmlDevice_t for use in NVML calls */
+};
+
+struct clock_reason {
+	unsigned long long mask;
+	char *short_name;
+	char *long_name;
+};
+
+unsigned short read_pci_id(char *device_path, char *field);
+char *read_nvidia_pci_address(char *device_path);
+void mmap_bar0(struct card_info *card);
+#endif

evga-card.c

@@ -6,19 +6,93 @@
 #include "evga-card.h"
 #include "icx3.h"
 
+struct gpu_pci_info {
+	char *card_name;
+	unsigned short vendor_id;
+	unsigned short device_id;
+	unsigned short subvendor_id;
+	unsigned short subdevice_id;
+};
+
+static struct gpu_pci_info evga_pci_ids[] =
+{
+	{"EVGA GeForce RTX 3060 Ti FTW3 Gaming"                , NVIDIA_VEN,   NVIDIA_RTX3060TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_GAMING_SUB_DEV            },
+	{"EVGA GeForce RTX 3060 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3060TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_ULTRA_SUB_DEV             },
+	{"EVGA GeForce RTX 3060 Ti FTW3 Ultra LHR"             , NVIDIA_VEN,   NVIDIA_RTX3060TI_LHR_DEV,   EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_ULTRA_KL_SUB_DEV          },
+	{"EVGA GeForce RTX 3070 Black Gaming"                  , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_BLACK_SUB_DEV                },
+	{"EVGA GeForce RTX 3070 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_GAMING_SUB_DEV               },
+	{"EVGA GeForce RTX 3070 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_SUB_DEV                },
+	{"EVGA GeForce RTX 3070 XC3 Ultra LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_LHR_SUB_DEV            },
+	{"EVGA GeForce RTX 3070 XC3 Ultra Gaming"              , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_GAMING_LHR_SUB_DEV     },
+	{"EVGA GeForce RTX 3070 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_FTW3_ULTRA_SUB_DEV               },
+	{"EVGA GeForce RTX 3070 FTW3 Ultra LHR"                , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_FTW3_ULTRA_LHR_SUB_DEV           },
+	{"EVGA GeForce RTX 3070 Ti XC3 Gaming"                 , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_GAMING_SUB_DEV             },
+	{"EVGA GeForce RTX 3070 Ti XC3 Ultra"                  , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_ULTRA_SUB_DEV              },
+	{"EVGA GeForce RTX 3070 Ti XC3 Ultra v2"               , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_ULTRA_V2_SUB_DEV           },
+	{"EVGA GeForce RTX 3070 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_FTW3_ULTRA_SUB_DEV             },
+	{"EVGA GeForce RTX 3070 Ti FTW3 Ultra v2"              , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_FTW3_ULTRA_V2_SUB_DEV          },
+	{"EVGA GeForce RTX 3080 XC3 Black"                     , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_BLACK_SUB_DEV                },
+	{"EVGA GeForce RTX 3080 XC3 Black LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_BLACK_LHR_SUB_DEV            },
+	{"EVGA GeForce RTX 3080 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_GAMING_SUB_DEV               },
+	{"EVGA GeForce RTX 3080 XC3 Gaming LHR"                , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_GAMING_LHR_SUB_DEV           },
+	{"EVGA GeForce RTX 3080 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_SUB_DEV                },
+	{"EVGA GeForce RTX 3080 XC3 Ultra LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_LHR_SUB_DEV            },
+	{"EVGA GeForce RTX 3080 XC3 Ultra Hybrid"              , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HYBRID_SUB_DEV         },
+	{"EVGA GeForce RTX 3080 XC3 Ultra Hybrid LHR"          , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HYBRID_LHR_SUB_DEV     },
+	{"EVGA GeForce RTX 3080 XC3 Ultra Hydro Copper"        , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HC_SUB_DEV             },
+	{"EVGA GeForce RTX 3080 FTW3 Gaming"                   , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_GAMING_SUB_DEV              },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_SUB_DEV               },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra v2 LHR"             , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_LHR_V2_SUB_DEV        },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra LHR"                , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_LHR_SUB_DEV           },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid"             , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_SUB_DEV        },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid LHR"         , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_LHR_SUB_DEV    },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid Gaming LHR"  , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_GAMING_SUB_DEV },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hydro Copper"       , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HC_SUB_DEV            },
+	{"EVGA GeForce RTX 3080 XC3 Ultra 12G"                 , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_XC3_ULTRA_SUB_DEV            },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra 12GB"               , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_FTW3_ULTRA_SUB_DEV           },
+	{"EVGA GeForce RTX 3080 FTW3 Ultra Hydro Copper 12G"   , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_FTW3_ULTRA_HC_SUB_DEV        },
+	{"EVGA GeForce RTX 3080 Ti XC3 Gaming"                 , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_SUB_DEV             },
+	{"EVGA GeForce RTX 3080 Ti XC3 Ultra Gaming"           , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_ULTRA_GAMING_SUB_DEV       },
+	{"EVGA GeForce RTX 3080 Ti XC3 Gaming Hybrid"          , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_HYBRID_SUB_DEV      },
+	{"EVGA GeForce RTX 3080 Ti XC3 Gaming Hydro Copper"    , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_HC_SUB_DEV          },
+	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_SUB_DEV             },
+	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra Hybrid"          , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_HYBRID_SUB_DEV      },
+	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra Hydro Copper"    , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_HC_SUB_DEV          },
+	{"EVGA GeForce RTX 3090 XC3 Black"                     , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_BLACK_SUB_DEV                },
+	{"EVGA GeForce RTX 3090 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_GAMING_SUB_DEV               },
+	{"EVGA GeForce RTX 3090 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_SUB_DEV                },
+	{"EVGA GeForce RTX 3090 XC3 Ultra Hybrid"              , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_HYBRID_SUB_DEV         },
+	{"EVGA GeForce RTX 3090 XC3 Ultra Hydro Copper"        , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_HC_SUB_DEV             },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_SUB_DEV               },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra v2"                 , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_V2_SUB_DEV            },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra v3"                 , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_V3_SUB_DEV            },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra Hybrid"             , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_HYBRID_SUB_DEV        },
+	{"EVGA GeForce RTX 3090 FTW3 Ultra Hydro Copper"       , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_HC_SUB_DEV            },
+	{"EVGA GeForce RTX 3090 K|NGP|N Hybrid"                , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_KINGPIN_HYBRID_SUB_DEV           },
+	{"EVGA GeForce RTX 3090 K|NGP|N Hydro Copper"          , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_KINGPIN_HC_SUB_DEV               },
+	{"EVGA GeForce RTX 3090 Ti FTW3 Black Gaming"          , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_BLACK_SUB_DEV             },
+	{"EVGA GeForce RTX 3090 Ti FTW3 Gaming"                , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_GAMING_SUB_DEV            },
+	{"EVGA GeForce RTX 3090 Ti FTW3 Ultra Gaming"          , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_ULTRA_GAMING_SUB_DEV      }
+};
+
+
+
 /* Search all i2c device files for ones are on a PCI device of a supported GPU,
    and respond with the correct iCX3 version information */
-int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
+int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus, int i2c_bus)
 {
-	char i2c_devices_path[NAME_MAX];
-	char device_path[NAME_MAX];
-	char dev_file[NAME_MAX];
+	char i2c_devices_path[PATH_MAX];
+	char device_path[PATH_MAX];
+	char dev_file[PATH_MAX];
+	char pci_path[PATH_MAX];
+	char *pci_addr;
 
 	FILE *test_fd;
 	DIR *dir;
 	struct dirent *ent;
 	
 	int num_gpus = 0;
+	int current_i2c_bus = -1;
 	unsigned short pci_vendor, pci_device, pci_subsystem_vendor, pci_subsystem_device = 0;
 	
 	/* Start looking for I2C adapters in /sys/bus/i2c/devices/ */
@@ -36,14 +110,30 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
 		if(strncmp(ent->d_name, "i2c-", 4) != 0) 
 			continue;
 		
+		/* Only probe the specific device given (if provided) */
+		if (i2c_bus >= 0) {
+			sscanf(ent->d_name, "i2c-%i", &current_i2c_bus);
+			if (current_i2c_bus != i2c_bus)
+				continue;
+		}
+		
 		strcpy(device_path, i2c_devices_path);
 		strcat(device_path, ent->d_name);
 		
+		/* Only check Nvidia devices */
+		pci_addr = read_nvidia_pci_address(device_path);
+		if (pci_addr == NULL)
+			continue;
+		
 		/* Read the PCI info for the underlying device */
-		pci_vendor           = read_pci_id(device_path, "/device/vendor");
-		pci_device           = read_pci_id(device_path, "/device/device");
-		pci_subsystem_vendor = read_pci_id(device_path, "/device/subsystem_vendor");
-		pci_subsystem_device = read_pci_id(device_path, "/device/subsystem_device");
+		strcat(device_path, "/../");
+		if (realpath(device_path, pci_path) == NULL)
+			continue;
+		
+		pci_vendor           = read_pci_id(pci_path, "/vendor");
+		pci_device           = read_pci_id(pci_path, "/device");
+		pci_subsystem_vendor = read_pci_id(pci_path, "/subsystem_vendor");
+		pci_subsystem_device = read_pci_id(pci_path, "/subsystem_device");
 		
 		/* See if it's a matching device for a supported EVGA card */
 		for (int i = 0; i < (sizeof(evga_pci_ids) / sizeof(struct gpu_pci_info)); i++) {
@@ -56,12 +146,15 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
 				/* Matched all PCI IDs, check for good firmware read */
 				strcpy(dev_file, "/dev/");
 				strcat(dev_file, ent->d_name);
-				if (check_for_icx3(dev_file)) {
+				infos[num_gpus].i2c_dev_path = dev_file;
+				if (icx3_init(&infos[num_gpus]) > 0) {
 					/* Write our card info into the provided struct array */
 					infos[num_gpus].card_name = evga_pci_ids[i].card_name;
-					infos[num_gpus].pci_id = read_nvidia_pci_address(device_path);
-					infos[num_gpus].i2c_dev_path = calloc(strlen(dev_file) + 1, sizeof(char));
-					strcpy(infos[num_gpus].i2c_dev_path, dev_file);
+					infos[num_gpus].pci_id = pci_addr;
+					infos[num_gpus].pci_dev_path = strdup(pci_path);
+					infos[num_gpus].pci_device_id = pci_device;
+					infos[num_gpus].i2c_dev_path = strdup(dev_file);
+					mmap_bar0(&infos[num_gpus]);
 					num_gpus++;
 					break;
 				}
@@ -70,53 +163,7 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
 		
 		if (num_gpus == max_gpus)
 			break;
-		
 	}
 	
 	return num_gpus;
-}
-
-/* Read a 16-bit unsigned int from a pci id file */
-unsigned short read_pci_id(char *device_path, char *field)
-{
-	char buf[16];
-	char file_path[NAME_MAX];
-	
-	strcpy(file_path, device_path);
-	strcat(file_path, field);
-	
-	FILE *fp = fopen(file_path, "r");
-	
-	if (fp == NULL)
-		return 0;
-		
-	if (fgets(buf, sizeof(buf), fp) == NULL) {
-		fclose(fp);
-		return 0;
-	}
-	
-	fclose(fp);
-	
-	return (unsigned short)strtoul(buf, NULL, 16);
-}
-
-char *read_nvidia_pci_address(char *device_path)
-{
-	char file_path[NAME_MAX];
-	
-	char *ret = calloc(16 + 1, sizeof(char)); /* assuming pci ids could look as large as 00000000:0C:00.0 */
-	
-	strcpy(file_path, device_path);
-	strcat(file_path, "/name");
-	
-	FILE *fp = fopen(file_path, "r");
-	
-	if (fp == NULL)
-		return NULL;
-	
-	fscanf(fp, "NVIDIA i2c adapter %*u at %16s", ret);
-	
-	fclose(fp);
-	
-	return ret;
-}
+}

evga-card.h

@@ -1,23 +1,7 @@
-#define NVIDIA_VEN                                      0x10DE
+#ifndef EVGA_CARD_H
+#define EVGA_CARD_H
 
-#define NVIDIA_RTX3060_DEV                              0x2503
-#define NVIDIA_RTX3060_GA104_DEV                        0x2487
-#define NVIDIA_RTX3060_GA106_DEV                        0x2503
-#define NVIDIA_RTX3060_LHR_DEV                          0x2504
-#define NVIDIA_RTX3060TI_DEV                            0x2486
-#define NVIDIA_RTX3060TI_GDDR6X_DEV                     0x24C9
-#define NVIDIA_RTX3060TI_LHR_DEV                        0x2489
-#define NVIDIA_RTX3060TI_V1_LHR_DEV                     0x2414
-#define NVIDIA_RTX3070_DEV                              0x2484
-#define NVIDIA_RTX3070_LHR_DEV                          0x2488
-#define NVIDIA_RTX3070TI_DEV                            0x2482
-#define NVIDIA_RTX3070TI_GA102_DEV                      0x2207
-#define NVIDIA_RTX3080_DEV                              0x2206
-#define NVIDIA_RTX3080_LHR_DEV                          0x2216
-#define NVIDIA_RTX3080_12G_LHR_DEV                      0x220A
-#define NVIDIA_RTX3080TI_DEV                            0x2208
-#define NVIDIA_RTX3090_DEV                              0x2204
-#define NVIDIA_RTX3090TI_DEV                            0x2203
+#include "card.h"
 
 #define EVGA_SUB_VEN                                    0x3842
 
@@ -80,81 +64,6 @@
 #define EVGA_RTX3090TI_FTW3_GAMING_SUB_DEV              0x4983
 #define EVGA_RTX3090TI_FTW3_ULTRA_GAMING_SUB_DEV        0x4985
 
-struct card_info {
-	char *card_name;
-	char *pci_id;
-	char *i2c_dev_path;
-};
+int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus, int i2c_bus);
 
-struct gpu_pci_info {
-	char *card_name;
-	unsigned short vendor_id;
-	unsigned short device_id;
-	unsigned short subvendor_id;
-	unsigned short subdevice_id;
-};
-
-static struct gpu_pci_info evga_pci_ids[] =
-{
-	{"EVGA GeForce RTX 3060 Ti FTW3 Gaming"                , NVIDIA_VEN,   NVIDIA_RTX3060TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_GAMING_SUB_DEV            },
-	{"EVGA GeForce RTX 3060 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3060TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_ULTRA_SUB_DEV             },
-	{"EVGA GeForce RTX 3060 Ti FTW3 Ultra LHR"             , NVIDIA_VEN,   NVIDIA_RTX3060TI_LHR_DEV,   EVGA_SUB_VEN,   EVGA_RTX3060TI_FTW3_ULTRA_KL_SUB_DEV          },
-	{"EVGA GeForce RTX 3070 Black Gaming"                  , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_BLACK_SUB_DEV                },
-	{"EVGA GeForce RTX 3070 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_GAMING_SUB_DEV               },
-	{"EVGA GeForce RTX 3070 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_SUB_DEV                },
-	{"EVGA GeForce RTX 3070 XC3 Ultra LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_LHR_SUB_DEV            },
-	{"EVGA GeForce RTX 3070 XC3 Ultra Gaming"              , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_XC3_ULTRA_GAMING_LHR_SUB_DEV     },
-	{"EVGA GeForce RTX 3070 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3070_DEV,         EVGA_SUB_VEN,   EVGA_RTX3070_FTW3_ULTRA_SUB_DEV               },
-	{"EVGA GeForce RTX 3070 FTW3 Ultra LHR"                , NVIDIA_VEN,   NVIDIA_RTX3070_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3070_FTW3_ULTRA_LHR_SUB_DEV           },
-	{"EVGA GeForce RTX 3070 Ti XC3 Gaming"                 , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_GAMING_SUB_DEV             },
-	{"EVGA GeForce RTX 3070 Ti XC3 Ultra"                  , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_ULTRA_SUB_DEV              },
-	{"EVGA GeForce RTX 3070 Ti XC3 Ultra v2"               , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_XC3_ULTRA_V2_SUB_DEV           },
-	{"EVGA GeForce RTX 3070 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_FTW3_ULTRA_SUB_DEV             },
-	{"EVGA GeForce RTX 3070 Ti FTW3 Ultra v2"              , NVIDIA_VEN,   NVIDIA_RTX3070TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3070TI_FTW3_ULTRA_V2_SUB_DEV          },
-	{"EVGA GeForce RTX 3080 XC3 Black"                     , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_BLACK_SUB_DEV                },
-	{"EVGA GeForce RTX 3080 XC3 Black LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_BLACK_LHR_SUB_DEV            },
-	{"EVGA GeForce RTX 3080 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_GAMING_SUB_DEV               },
-	{"EVGA GeForce RTX 3080 XC3 Gaming LHR"                , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_GAMING_LHR_SUB_DEV           },
-	{"EVGA GeForce RTX 3080 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_SUB_DEV                },
-	{"EVGA GeForce RTX 3080 XC3 Ultra LHR"                 , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_LHR_SUB_DEV            },
-	{"EVGA GeForce RTX 3080 XC3 Ultra Hybrid"              , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HYBRID_SUB_DEV         },
-	{"EVGA GeForce RTX 3080 XC3 Ultra Hybrid LHR"          , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HYBRID_LHR_SUB_DEV     },
-	{"EVGA GeForce RTX 3080 XC3 Ultra Hydro Copper"        , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_XC3_ULTRA_HC_SUB_DEV             },
-	{"EVGA GeForce RTX 3080 FTW3 Gaming"                   , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_GAMING_SUB_DEV              },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_SUB_DEV               },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra v2 LHR"             , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_LHR_V2_SUB_DEV        },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra LHR"                , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_LHR_SUB_DEV           },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid"             , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_SUB_DEV        },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid LHR"         , NVIDIA_VEN,   NVIDIA_RTX3080_LHR_DEV,     EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_LHR_SUB_DEV    },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hybrid Gaming LHR"  , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HYBRID_GAMING_SUB_DEV },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hydro Copper"       , NVIDIA_VEN,   NVIDIA_RTX3080_DEV,         EVGA_SUB_VEN,   EVGA_RTX3080_FTW3_ULTRA_HC_SUB_DEV            },
-	{"EVGA GeForce RTX 3080 XC3 Ultra 12G"                 , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_XC3_ULTRA_SUB_DEV            },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra 12GB"               , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_FTW3_ULTRA_SUB_DEV           },
-	{"EVGA GeForce RTX 3080 FTW3 Ultra Hydro Copper 12G"   , NVIDIA_VEN,   NVIDIA_RTX3080_12G_LHR_DEV, EVGA_SUB_VEN,   EVGA_RTX3080_12G_FTW3_ULTRA_HC_SUB_DEV        },
-	{"EVGA GeForce RTX 3080 Ti XC3 Gaming"                 , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_SUB_DEV             },
-	{"EVGA GeForce RTX 3080 Ti XC3 Ultra Gaming"           , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_ULTRA_GAMING_SUB_DEV       },
-	{"EVGA GeForce RTX 3080 Ti XC3 Gaming Hybrid"          , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_HYBRID_SUB_DEV      },
-	{"EVGA GeForce RTX 3080 Ti XC3 Gaming Hydro Copper"    , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_XC3_GAMING_HC_SUB_DEV          },
-	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra"                 , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_SUB_DEV             },
-	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra Hybrid"          , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_HYBRID_SUB_DEV      },
-	{"EVGA GeForce RTX 3080 Ti FTW3 Ultra Hydro Copper"    , NVIDIA_VEN,   NVIDIA_RTX3080TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3080TI_FTW3_ULTRA_HC_SUB_DEV          },
-	{"EVGA GeForce RTX 3090 XC3 Black"                     , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_BLACK_SUB_DEV                },
-	{"EVGA GeForce RTX 3090 XC3 Gaming"                    , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_GAMING_SUB_DEV               },
-	{"EVGA GeForce RTX 3090 XC3 Ultra"                     , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_SUB_DEV                },
-	{"EVGA GeForce RTX 3090 XC3 Ultra Hybrid"              , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_HYBRID_SUB_DEV         },
-	{"EVGA GeForce RTX 3090 XC3 Ultra Hydro Copper"        , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_XC3_ULTRA_HC_SUB_DEV             },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra"                    , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_SUB_DEV               },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra v2"                 , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_V2_SUB_DEV            },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra v3"                 , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_V3_SUB_DEV            },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra Hybrid"             , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_HYBRID_SUB_DEV        },
-	{"EVGA GeForce RTX 3090 FTW3 Ultra Hydro Copper"       , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_FTW3_ULTRA_HC_SUB_DEV            },
-	{"EVGA GeForce RTX 3090 K|NGP|N Hybrid"                , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_KINGPIN_HYBRID_SUB_DEV           },
-	{"EVGA GeForce RTX 3090 K|NGP|N Hydro Copper"          , NVIDIA_VEN,   NVIDIA_RTX3090_DEV,         EVGA_SUB_VEN,   EVGA_RTX3090_KINGPIN_HC_SUB_DEV               },
-	{"EVGA GeForce RTX 3090 Ti FTW3 Black Gaming"          , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_BLACK_SUB_DEV             },
-	{"EVGA GeForce RTX 3090 Ti FTW3 Gaming"                , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_GAMING_SUB_DEV            },
-	{"EVGA GeForce RTX 3090 Ti FTW3 Ultra Gaming"          , NVIDIA_VEN,   NVIDIA_RTX3090TI_DEV,       EVGA_SUB_VEN,   EVGA_RTX3090TI_FTW3_ULTRA_GAMING_SUB_DEV      }
-};
-
-int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus);
-unsigned short read_pci_id(char *device_path, char *field);
-char *read_nvidia_pci_address(char *device_path);
+#endif

evga-icx.c

@@ -5,13 +5,36 @@
 #include <fcntl.h>
 #include <stdlib.h>
 
+#ifdef USE_NVML
+	#include "nvidia-sensors.h"
+#endif
+
+#ifdef USE_DCGM
+	#include "nvidia-dcgm.h"
+#endif
+
+#include "blackwell-sensors.h"
+#include "gddr6.h"
 #include "icx3.h"
 #include "evga-card.h"
+#include "board-sensors.h"
+#include "zen3-rapl.h"
 
 #define MAX_GPUS 16
+#define MAX_BOARD_SENSORS 256
+#define HEADER_COLOR_START "\x1b[36m"
+#define HEADER_COLOR_END "\x1b[39m"
+
+static char *header_start = "";
+static char *header_end = "";
+
+static int zen3_rapl_sensor = -1;	/* Board sensor number for the RAPL sensor */
+static int compact = 0;				/* Compact one-line per GPU display */
+static int no_reasons = 0;			/* Don't probe or display NVML clock reasons */
 
 static const char helpstring[] = "Available options:\n"
-                                 "--gpu N      : Control only GPU N instead of all supported cards\n"
+								 "--i2c N      : Only probe I2C bus N instead of all (may help with stuttering or freezing when probing I2C devices)\n"
+								 "--gpu N      : Control only GPU N instead of all supported cards\n"
 								 "--fan SPEED  : Set all fans at once to SPEED (see below)\n"
 								 "--fanN SPEED : Set fan N (0-3) to SPEED\n"
 								 "    SPEED may be one of the following:\n"
@@ -19,21 +42,41 @@ static const char helpstring[] = "Available options:\n"
 								 "      N to set the fan to that manual % speed\n"
 								 "      [+/-]N to set that fan to an RPM offset from the GPU-controlled speed\n"
 								 "--reset      : Reset all fans to their default mode\n"
-								 "--sensors    : Print sensor readings even if setting a fan speed \n";
+								 "--sensors    : Print sensor readings even if setting a fan speed \n"
+								 "--compact    : Print sensor reading in a compact one-line per card format\n"
+								 "--watch N    : Keep printing output every N seconds\n"
+								 "--overwrite  : Overwrite previously displayed info with --watch and --compact instead of continuously logging\n"
+								 "--color      : Print headers in color in --compact mode for better readability\n"
+								 "--no-reasons : Do not query NVML for clock reasons (can cause stuttering)\n"
+								 "--board      : Also print temperatures from the CPU, motherboard, and other sensors\n";
 								 
 void print_gpu_info(int gpu_num, struct card_info gpus[]);
+void print_board_info(struct hwmon_avail_sensor *board_sensors, int num_sensors);
 
 int main (int argc, char **argv)
 {
 	struct card_info gpus[MAX_GPUS];
-	int gpu_count;
-	int print_info = 0;
-	int gpu_num = -1;    /* Card to control */
+	struct hwmon_avail_sensor board_sensors[MAX_BOARD_SENSORS];
+	int gpu_count, board_sensor_count;
+	int print_info = 0;			
+	int gpu_num = -1;				/* Card to control */
+	int i2c_bus = -1;				/* Specific i2c bus to probe instead of all */
+	int overwrite = 0;				/* Overwrite printed console info in compact mode */		
+	unsigned int watch = 0;			/* Refresh display every this many seconds */
+	int print_board_sensors = 0;	/* Print CPU/motherbord/other sensors as well */
 	char *fan_speed[ICX3_MAX_FANS] = {NULL}; 
 	
 	/* Input parsing */
 	for (int i = 1; i < argc; i++){
-		if (strcmp(argv[i], "--gpu") == 0) {
+		if (strcmp(argv[i], "--i2c") == 0) {
+			i++;
+			if (i < argc) {
+				i2c_bus = atoi(argv[i]);
+			} else {
+				printf(helpstring);
+				return -1;
+			}
+		} else if (strcmp(argv[i], "--gpu") == 0) {
 			i++;
 			if (i < argc) {
 				gpu_num = atoi(argv[i]);
@@ -65,6 +108,25 @@ int main (int argc, char **argv)
 				fan_speed[j] = "auto";
 		} else if (strcmp(argv[i], "--sensors") == 0) {
 			print_info = 1;
+		} else if (strcmp(argv[i], "--compact") == 0) {
+			compact = 1;
+		} else if (strcmp(argv[i], "--watch") == 0) {
+			i++;
+			if (i < argc) {
+				watch = atoi(argv[i]);
+			} else {
+				printf(helpstring);
+				return -1;
+			}
+		} else if (strcmp(argv[i], "--overwrite") == 0) {
+			overwrite = 1;
+		} else if (strcmp(argv[i], "--color") == 0) {
+			header_start = HEADER_COLOR_START;
+			header_end = HEADER_COLOR_END;
+		} else if (strcmp(argv[i], "--no-reasons") == 0) {
+			no_reasons = 1;
+		} else if (strcmp(argv[i], "--board") == 0) {
+			print_board_sensors = 1;
 		} else {
 			printf(helpstring);
 			return 0;
@@ -80,56 +142,263 @@ int main (int argc, char **argv)
 		}
 	}
 	
-	gpu_count = find_evga_gpu_i2cs(gpus, MAX_GPUS);
+	/* Don't use overwrite mode unless set to compact (we can't tell how many lines the output will be per GPU) */
+	if (overwrite && !compact)
+		overwrite = 0;
 	
+	/* Scan for supported GPUs */
+	gpu_count = find_evga_gpu_i2cs(gpus, MAX_GPUS, i2c_bus);
+	/* Scan for supported Blackwell GPUs */
+	gpu_count = find_blackwell_gpus(gpus, gpu_count, MAX_GPUS);
+
+	/* Check for no GPUs found or other errors */
 	if (gpu_count == -1) {
 		printf("Error scanning I2C devices\n");
 		return -1;
 	} else if (gpu_count == 0) {
 		printf("No supported GPUs found.\nAre you root or do you have udev access to i2c devices?\nDo you need to run `modprobe i2c-dev`?\n");
-		return -1;
 	}
-	
+	/* Check for invalid GPUs */
 	if (gpu_num > gpu_count - 1) {
 		printf("Invalid GPU number specified (%d, max %d)\n", gpu_num, gpu_count - 1);
 		return -1;
 	}
 		
+	/* Scan for motherboard/CPU/other sensors */
+	if (print_board_sensors) {
+		board_sensor_count = find_board_sensors(board_sensors, MAX_BOARD_SENSORS);
+		if (init_rapl() && board_sensor_count < MAX_BOARD_SENSORS) {
+			board_sensors[board_sensor_count] = rapl_sensor;
+			zen3_rapl_sensor = board_sensor_count;
+			board_sensor_count++;
+		}
+	}
 	
 	/* execute fan commands */
 	if (gpu_num == -1) {
 		for (int i = 0; i < gpu_count; i++){
 			for (int j = 0; j < ICX3_MAX_FANS; j++) {
 				if (fan_speed[j] != NULL)
-					set_fan(j, fan_speed[j], gpus[i].i2c_dev_path);
-					/* printf("gpu %d fan %d : %s\n", i, j, fan_speed[j]); */
+					set_fan(j, fan_speed[j], &gpus[i]);
 			}
 		}
 	} else if (gpu_num <= gpu_count - 1) {
 		for (int j = 0; j < ICX3_MAX_FANS; j++) {
 			if (fan_speed[j] != NULL)
-				set_fan(gpu_num, fan_speed[j], gpus[gpu_num].i2c_dev_path);
-				/* printf("gpu %d fan %d : %s\n", gpu_num, j, fan_speed[j]);*/
+				set_fan(gpu_num, fan_speed[j], &gpus[gpu_num]);
 		}
 	}
 	
+	/* NVML init */
+#ifdef USE_NVML
+    init_nvml();
+	for (int i = 0; i < gpu_count; i++) {
+		gpus[i].nvml_device = malloc(sizeof(nvmlDevice_t));
+		get_nvml_handle(&gpus[i]);
+	}
+#endif
+
+#ifdef USE_DCGM
+	init_dcgm();
+#endif
+
 	/* print sensor info */
 	if (print_info) {
-		if (gpu_num == -1) {
-			for (int i = 0; i < gpu_count; i++){
-				print_gpu_info(i, gpus);
-			}		
-		} else if (gpu_num <= gpu_count - 1) {
-			print_gpu_info(gpu_num, gpus);
-		}
+		do {
+			if (overwrite)
+				printf("\x1b[K");	/* Clear current console line */
+			
+			if (print_board_sensors)
+				print_board_info(board_sensors, board_sensor_count);
+
+			if (gpu_num == -1) {
+				/* No GPU specified on command line, loop over all supported GPUs */
+				for (int i = 0; i < gpu_count; i++){
+					if (i > 0)
+						printf("\n");
+					if (gpus[i].i2c_dev_path == NULL) {
+						print_blackwell_info(compact, overwrite, &gpus[i]);
+					} else {
+						print_gpu_info(i, &gpus[i]);
+					}
+				}		
+			} else if (gpu_num <= gpu_count - 1) {
+				print_gpu_info(gpu_num, &gpus[gpu_num]);
+			}
+			
+#ifdef USE_DCGM
+			if (compact && (gpu_count > 0))
+				printf("\n");
+			print_dcgm(compact, overwrite);
+#endif
+
+			if (!overwrite)
+				printf("\n"); /* Print line break at the end for continuous output */
+			
+			if (overwrite && compact) {
+				int total_lines = 0;
+				if (print_board_sensors) total_lines++;
+				if (gpu_count > 0) {
+					total_lines += (gpu_num == -1) ? gpu_count : 1;
+				}
+#ifdef USE_DCGM
+				total_lines++;
+#endif
+				printf("\x1b[1G");						/* Move cursor back to column 1 */
+				if (total_lines > 1)
+					printf("\x1b[%dA", total_lines - 1);	/* Move cursor back up to the top */
+			}
+			
+			fflush(stdout);
+			sleep(watch);
+		} while (watch > 0);
 	}
 	
+#ifdef USE_NVML
+	nvmlShutdown();
+#endif
+#ifdef USE_DCGM
+	shutdown_dcgm();
+#endif
 }
 
-void print_gpu_info(int gpu_num, struct card_info gpus[]) {
-	printf("#%d: %s (%s) @ %s\n", gpu_num, gpus[gpu_num].card_name, gpus[gpu_num].i2c_dev_path, gpus[gpu_num].pci_id);
-	print_icx3_fans(gpus[gpu_num].i2c_dev_path);
-	print_icx3_temps(gpus[gpu_num].i2c_dev_path);
+void print_board_info(struct hwmon_avail_sensor *board_sensors, int num_sensors)
+{
+	int printed_sensors = 0;
+	int current_sort_index = 0;
+	float current_reading = 0.0;
+	int good_reading = 0;
+	/* These allow us to 'summarize' units and categories by only printing them when they change */
+	char *last_short_name = NULL;
+	char *last_units = NULL;
+	
+	float rapl_power;
+	
+	for (int i = 0; i < (sizeof(hwmon_sensor_info) / sizeof(struct hwmon_sensor)); i++) {
+		
+		/* Inject our Zen RAPL power reading here */
+		if (strcmp(hwmon_sensor_info[i].driver_name, "zen-rapl") == 0) {
+			board_sensors[zen3_rapl_sensor].sort_index = current_sort_index;
+			board_sensors[zen3_rapl_sensor].sensor_info = &hwmon_sensor_info[i];
+		}
+		
+		for (int j=0; j < num_sensors; j++) {
+			struct hwmon_avail_sensor *current_sensor = &board_sensors[j];
+
+			/* Loop over all sensors, but only output those with the current sort index so they come out sort of sorted
+			Duplicates (e.g.) multiple NVMe will come out in whatever sort of order the directory listing happened to */
+			if (current_sensor->sort_index == current_sort_index) {
+				printed_sensors++;
+				
+				if (j == zen3_rapl_sensor) {
+					good_reading = 1;
+					current_reading = get_rapl_package_power();
+				} else {
+					good_reading = get_sensor_reading(current_sensor, &current_reading);
+				}
+				
+				if (!good_reading)
+					continue;
+				
+				if (compact) {
+					/* Print units if needed */
+					if (last_units != NULL && strcmp(current_sensor->sensor_info->units, last_units))
+						printf("%s", last_units);
+					
+					/* Print new section header if needed */
+					if (last_short_name == NULL || strcmp(current_sensor->sensor_info->short_name, last_short_name)) {
+						if (last_short_name != NULL) /* Spacer for all headings not the first one */
+							printf(" ");
+						printf("%s%s%s", header_start, current_sensor->sensor_info->short_name, header_end);
+					}
+					printf(" %3.0f", current_reading);
+					
+					last_short_name = current_sensor->sensor_info->short_name;
+					last_units = current_sensor->sensor_info->units;
+					
+				} else {
+					printf("%s%s: %+.1f%s\n", 
+						current_sensor->sensor_info->name_prefix,
+						current_sensor->sensor_name,
+						current_reading,
+						current_sensor->sensor_info->units);
+				}
+				
+			}
+		}
+		current_sort_index++;
+	}
+	
+	if (compact && last_units != NULL)
+		printf("%s", last_units);
+	
 	printf("\n");
 }
+
+void print_gpu_info(int gpu_num, struct card_info *gpu)
+{
+	if (compact) {
+		/* One line per GPU */
+		printf("%s#%d FAN%s", header_start, gpu_num, header_end);
+		print_icx3_fans_oneline(gpu);
+		printf("%s GPU%s", header_start, header_end);
+		
+#ifdef USE_NVML
+		printf(" %3d", get_nvml_temp(gpu));
+#endif
+		
+		float icx_temp_sensors[ICX3_NUM_TEMP_SENSORS] = {};
+		get_temp_sensors(icx_temp_sensors, gpu);
+		for (int i = 0; i < ICX3_NUM_TEMP_SENSORS; i++) {
+			if (i > 0 && strncmp(icx3_temp_sensor_names[i], icx3_temp_sensor_names[i-1], 3))
+				printf("%s %.3s%s", header_start, icx3_temp_sensor_names[i], header_end);
+			if (strncmp(icx3_temp_sensor_names[i], "MEM1", 4) == 0)
+				printf(" %3.0f", get_vram_temp(gpu));			/* Print the VRAM temp before the rest of the memory sensors */
+			printf(" %3.0f", icx_temp_sensors[i]);
+		}
+		
+		printf("%s HOT%s %3.0f", header_start, header_end, get_hotspot_temp(gpu));
+		
+		printf("°C ");
+		
+#ifdef USE_NVML
+		printf("%s MEM %s", header_start, header_end);
+		printf("%3d%%", get_nvml_mem_util(gpu));
+		if (!no_reasons) {
+			printf("%s CLK %s", header_start, header_end);
+			print_nvml_clock_reason(1, gpu);
+		}
+#endif
+
+
+	} else {
+		/* One line per GPU sensor */
+		printf("#%d: %s (%s) @ %s\n", gpu_num, gpu->card_name, gpu->i2c_dev_path, gpu->pci_id);
+		print_icx3_fans(gpu);
+		
+#ifdef USE_NVML
+		printf("GPU1: %+d°C\n", get_nvml_temp(gpu));
+#endif
+		float icx_temp_sensors[ICX3_NUM_TEMP_SENSORS] = {};
+		get_temp_sensors(icx_temp_sensors, gpu);
+		for (int i = 0; i < ICX3_NUM_TEMP_SENSORS; i++) {
+			if (strncmp(icx3_temp_sensor_names[i], "MEM1", 4) == 0)
+				printf("VRAM: +%.0f°C\n", get_vram_temp(gpu));			/* Print the VRAM temp before the rest of the memory sensors */
+			printf("%s: %+.1f°C\n",
+				icx3_temp_sensor_names[i],
+				icx_temp_sensors[i]);
+		}
+		
+		printf("HotSpot: +%.0f°C\n", get_hotspot_temp(gpu));
+
+#ifdef USE_NVML
+		printf("Mem util: %d%%\n", get_nvml_mem_util(gpu));
+		if (!no_reasons) {
+			printf("Clock reasons: ");
+			print_nvml_clock_reason(0, gpu);
+		}
+		printf("\n");
+#endif
+	}
+}
 	

gddr6.c

@@ -0,0 +1,48 @@
+#include <string.h>
+#include <stdio.h>
+
+#include "gddr6.h"
+
+static struct device_offset device_offset_info[] = 
+{
+	{.device_id = NVIDIA_RTX3090TI_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3090 Ti */
+	{.device_id = NVIDIA_RTX3090_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3090 */
+	{.device_id = NVIDIA_RTX3080TI_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 Ti */
+	{.device_id = NVIDIA_RTX3080_12G_LHR_DEV,	.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 12G LHR */
+	{.device_id = NVIDIA_RTX3080_LHR_DEV,		.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 LHR */
+	{.device_id = NVIDIA_RTX3080_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 */
+	{.device_id = NVIDIA_RTX3070TI_GA102_DEV,	.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 Ti GA102 TODO:check */
+	{.device_id = NVIDIA_RTX3070TI_DEV,			.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 Ti TODO:check */
+	{.device_id = NVIDIA_RTX3070_LHR_DEV,		.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 LHR */
+	{.device_id = NVIDIA_RTX3070_DEV,			.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 */
+};
+
+float get_vram_temp(struct card_info *card)
+{
+	if (card->bar0_mmap == NULL)
+		return 0.0;
+
+	for (int i = 0; i < sizeof(device_offset_info) / sizeof(struct device_offset); i++) {
+		if (card->pci_device_id == device_offset_info[i].device_id) {
+			unsigned int read_result = *((unsigned int *)((char *)card->bar0_mmap + device_offset_info[i].vram_offset));
+			return (float)((read_result & 0x00000fff) / 0x20);
+		}
+	}
+
+	return 0.0;
+}
+
+float get_hotspot_temp(struct card_info *card)
+{
+	if (card->bar0_mmap == NULL)
+		return 0.0;
+
+	for (int i = 0; i < sizeof(device_offset_info) / sizeof(struct device_offset); i++) {
+		if (card->pci_device_id == device_offset_info[i].device_id) {
+			unsigned int read_result = *((unsigned int *)((char *)card->bar0_mmap + device_offset_info[i].hotspot_offset));
+			return (float)((read_result >> 8) & 0xff);
+		}
+	}
+
+	return 0.0;
+}

gddr6.h

@@ -0,0 +1,13 @@
+#ifndef GDDR6_H
+#define GDDR6_H
+#include "evga-card.h"
+
+struct device_offset {
+	unsigned short device_id;
+	int vram_offset;
+	int hotspot_offset;
+};
+
+float get_vram_temp(struct card_info *card);
+float get_hotspot_temp(struct card_info *card);
+#endif

icx3.c

@@ -9,99 +9,68 @@
 
 #include "icx3.h"
 
-/* Check an I2C device file for an ICX3 controller, returns the product id, or ICX3_PRODUCT_NONE on bad read. */
-enum icx3_product_id check_for_icx3(char *i2c_dev_path)
+int icx3_init(struct card_info *card)
 {
 	char data[I2C_SMBUS_BLOCK_MAX] = {};
-	int fd, read_result;
+	int read_result;
 	struct icx3_info *temp_info;
 	
-	fd = open_i2c_dev(i2c_dev_path);
+	int fd = open(card->i2c_dev_path, O_RDONLY);
+	
 	if (fd == -1)
-		return ICX3_PRODUCT_NONE;
+		return -1;
+	
+	if (ioctl(fd, I2C_SLAVE, ICX3_I2C_ADDR) < 0) {
+		close(fd);
+		return -1;
+	}
 	
 	read_result = i2c_smbus_read_i2c_block_data(fd, ICX3_REG_READINFORMATION, ICX3_READINFORMATION_SIZE, data);
-	close(fd);
 	if (read_result == ICX3_READINFORMATION_SIZE) {
 		temp_info = (struct icx3_info*)&data;
-		if (temp_info->slave_address == ICX3_I2C_ADDR)
-			return temp_info->product_id;
+		if (temp_info->slave_address == ICX3_I2C_ADDR) {
+			card->product_id = temp_info->product_id;
+			card->i2c_fd = fd;
+			return fd;
+		} else {
+			close(fd);
+			return -1;
+		}
 	}
-
-	return ICX3_PRODUCT_NONE;
+	
+	return -1;
 }
 
-void print_icx3_fans(char *i2c_dev_path)
+void print_icx3_fans(struct card_info *card)
 {
-	char data[I2C_SMBUS_BLOCK_MAX] = {};
-	int fd, read_result;
+	struct icx3_fan_control fans[ICX3_MAX_FANS]; 
+	get_fan_status(fans, card);
 	
-	struct icx3_fan_control *fan_status; 
-	
-	/* First thing is to check for the product ID to figure out how many fans we have */
-	char fans_avail[ICX3_MAX_FANS] = {0};
-	get_available_fans(i2c_dev_path, fans_avail);
-	
-	fd = open_i2c_dev(i2c_dev_path);
-	if (fd == -1)
-		return;
-	
-	for (int i=0; i < ICX3_MAX_FANS; i++) {
-		if (!fans_avail[i])
-			continue;
-		
-		read_result = i2c_smbus_read_i2c_block_data(fd, ICX3_REG_FANCONTROL + i, ICX3_FANCONTROL_SIZE, data);
-		if (read_result != ICX3_FANCONTROL_SIZE) {
-			close(fd);
-			return;
-		}
-		fan_status = (struct icx3_fan_control*) &data;
+	for (int i=0; i < ICX3_MAX_FANS; i++) {		
 		printf("%s: %d RPM (%d/%d%%, %s)\n",
 			icx3_fan_names[i],
-			fan_status->rpm_status,
-			fan_status->duty_status,
-			fan_status->duty,
-			icx3_fan_mode_names[fan_status->fanmode]
+			fans[i].rpm_status,
+			fans[i].duty_status,
+			fans[i].duty,
+			icx3_fan_mode_names[fans[i].fanmode]
 			);
 	}
-	
-	close(fd);
 }
 
-void print_icx3_temps(char *i2c_dev_path)
+void print_icx3_fans_oneline(struct card_info *card)
 {
-	char data[I2C_SMBUS_BLOCK_MAX] = {};
-	int fd, read_result;
-	struct icx3_temp_sensors *temp_sensors;
+	struct icx3_fan_control fans[ICX3_MAX_FANS]; 
+	get_fan_status(fans, card);
 	
-	fd = open_i2c_dev(i2c_dev_path);
-	if (fd == -1)
-		return;
-	
-	read_result = i2c_smbus_read_i2c_block_data(fd, ICX3_REG_TEMPSENSOR, ICX3_TEMPSENSOR_SIZE, data);
-	if (read_result != ICX3_TEMPSENSOR_SIZE) {
-		close(fd);
-		return;
-	}
-	temp_sensors = (struct icx3_temp_sensors*) &data;
-	
-	float cur_temp;
-	short cur_data;
-	
-	for (int i=0; i<ICX3_NUM_TEMP_SENSORS; i++) {
-		/* endian swap */
-		cur_data = (short)(temp_sensors->data[2*i+1] << 8) | (short)(temp_sensors->data[2*i]);
-		/* temp is reported in tenths of deg C */
-		cur_temp = (float)cur_data/10;
-		printf("%s: %.1f C\n",
-		icx3_temp_sensor_names[i],
-		cur_temp);
+	for (int i=0; i < ICX3_MAX_FANS; i++) {		
+		printf(" %3d", fans[i].duty_status);
 	}
+	printf("%%");
 }
 
-void get_available_fans(char *i2c_dev_path, char *fans_avail)
+void get_available_fans(char *fans_avail, struct card_info *card)
 {
-	int product_id = check_for_icx3(i2c_dev_path);
+	int product_id = card->product_id;
 	
 	/* From ICX3TotalFanCtrl.cs */
 	switch(product_id) {
@@ -155,54 +124,77 @@ void get_available_fans(char *i2c_dev_path, char *fans_avail)
 	}
 }
 
-int open_i2c_dev(char *i2c_dev_path)
+void get_fan_status(struct icx3_fan_control *fans, struct card_info *card)
 {
-	int fd = open(i2c_dev_path, O_RDONLY);
+	char data[I2C_SMBUS_BLOCK_MAX] = {};
+	int read_result;
 	
-	/* Error silently here because we should have already checked we can read the i2c in check_for_icx3() */
-	if (fd == -1)
-		return -1;
-	
-	if (ioctl(fd, I2C_SLAVE, ICX3_I2C_ADDR) < 0) {
-		close(fd);
-		return -1;
+	char fans_avail[ICX3_MAX_FANS] = {0};
+	get_available_fans(fans_avail, card);
+		
+	for (int i=0; i < ICX3_MAX_FANS; i++) {
+		if (!fans_avail[i]) {
+			continue;
+		}
+		
+		read_result = i2c_smbus_read_i2c_block_data(card->i2c_fd, ICX3_REG_FANCONTROL + i, ICX3_FANCONTROL_SIZE, data);
+		if (read_result != ICX3_FANCONTROL_SIZE) {
+			return;
+		}
+		memcpy(&fans[i], &data, sizeof(struct icx3_fan_control));
 	}
-	return fd;
 }
 
-void enable_write(int enable, char *i2c_dev_path)
+void get_temp_sensors(float *temps, struct card_info *card)
 {
-	int fd = open_i2c_dev(i2c_dev_path);
-	if (fd == -1)
+	char data[I2C_SMBUS_BLOCK_MAX] = {};
+	int read_result;
+	struct icx3_temp_sensors *temp_sensors;
+		
+	read_result = i2c_smbus_read_i2c_block_data(card->i2c_fd, ICX3_REG_TEMPSENSOR, ICX3_TEMPSENSOR_SIZE, data);
+	if (read_result != ICX3_TEMPSENSOR_SIZE)
 		return;
+	temp_sensors = (struct icx3_temp_sensors*) &data;
 	
+	float cur_temp;
+	short cur_data;
+	
+	for (int i=0; i<ICX3_NUM_TEMP_SENSORS; i++) {
+		/* endian swap */
+		cur_data = (short)(temp_sensors->data[2*i+1] << 8) | (short)(temp_sensors->data[2*i]);
+		/* temp is reported in tenths of deg C */
+		temps[i] = (float)cur_data/10;
+	}
+}
+
+void enable_write(int enable, struct card_info *card)
+{	
 	unsigned char *data;
 	if (enable)
 		data = icx3_write_enable;
 	else
 		data = icx3_write_disable;
 	/* Enable or disable write */
-    i2c_smbus_write_i2c_block_data(fd, ICX3_REG_ENABLEWRITE, ICX3_ENABLEWRITE_SIZE, data);
+    i2c_smbus_write_i2c_block_data(card->i2c_fd, ICX3_REG_ENABLEWRITE, ICX3_ENABLEWRITE_SIZE, data);
 	
 	/* Read back the result to verify */
 	unsigned char read_result[ICX3_ENABLEWRITE_SIZE];
 	int write_ok = 0;
-	i2c_smbus_read_i2c_block_data(fd, ICX3_REG_ENABLEWRITE, ICX3_ENABLEWRITE_SIZE, read_result);
+	i2c_smbus_read_i2c_block_data(card->i2c_fd, ICX3_REG_ENABLEWRITE, ICX3_ENABLEWRITE_SIZE, read_result);
 	if (enable)
 		write_ok = (read_result[1] == 0xFC);
 	else
 		write_ok = (read_result[1] == 0xFE);
 	
 	if (!write_ok)
-		printf("Unable to enable/disable write on %s\n", i2c_dev_path);
-
+		printf("Unable to enable/disable write on %s\n", card->i2c_dev_path);
 }
 
 /* Sets a given fan on the GPU according to a string setting
 'auto' resets the fan to default (fans 1-2 go to GPU control, 3-4 to +0 RPM offset from GPU control)
 A number without a sign (e.g. 50) manually sets the fan to a given % duty cycle
 A number with a sign (e.g. +500) sets the fan to run at an RPM offset from GPU control */
-void set_fan(int fan, char *setting, char *i2c_dev_path)
+void set_fan(int fan, char *setting, struct card_info *card)
 {
 	char fans_avail[ICX3_MAX_FANS] = {0};
 	
@@ -213,15 +205,11 @@ void set_fan(int fan, char *setting, char *i2c_dev_path)
 	int write_result;
 	
 	/* Check to make sure we're setting a valid fan */
-	get_available_fans(i2c_dev_path, fans_avail);
+	get_available_fans(fans_avail, card);
 	if (fans_avail[fan] == 0) {
 		printf("Fan %d does not exist on this card \n", fan);
 		return;
 	}
-
-	int fp = open_i2c_dev(i2c_dev_path);
-	if (fp == -1)
-		return;
 	
 	fan_control.length = ICX3_FANCONTROL_SIZE - 1;
 	if (strcmp(setting, "auto") == 0) {
@@ -240,16 +228,15 @@ void set_fan(int fan, char *setting, char *i2c_dev_path)
 		fan_control.duty = atoi(setting);
 	}
 
-	enable_write(1, i2c_dev_path);
+	enable_write(1, card);
 	
-    i2c_smbus_write_i2c_block_data(fp, reg, ICX3_FANCONTROL_SIZE, (char *)&fan_control);
+    i2c_smbus_write_i2c_block_data(card->i2c_fd, reg, ICX3_FANCONTROL_SIZE, (char *)&fan_control);
 
 	/* Read back data and verify we set the fan properly */
-    i2c_smbus_read_i2c_block_data(fp, reg, ICX3_FANCONTROL_SIZE, (char *)&fan_readback);
+    i2c_smbus_read_i2c_block_data(card->i2c_fd, reg, ICX3_FANCONTROL_SIZE, (char *)&fan_readback);
 	if (fan_readback.fanmode    != fan_control.fanmode ||
 		fan_readback.rpm_offset != fan_control.rpm_offset ||
 		fan_readback.duty       != fan_control.duty)
-		printf("Error setting fan %d on %s\n", fan, i2c_dev_path);
-	close(fp);
+		printf("Error setting fan %d on %s\n", fan, card->i2c_dev_path);
 }
 

icx3.h

@@ -1,3 +1,9 @@
+#ifndef ICX3_H
+#define ICX3_H
+
+#include <stdint.h>
+#include "evga-card.h"
+
 #define ICX3_I2C_ADDR 0x2D
 
 #define ICX3_REG_FANCONTROL 80
@@ -17,26 +23,26 @@
 #define ICX3_WRITE_DISABLED 0xFE
 
 struct icx3_fan_control {
-	unsigned char length;
-	unsigned char fanmode;
-	unsigned short rpm_offset;
-	unsigned char duty;
-	unsigned char duty_status;
-	unsigned short rpm_status;
+	uint8_t length;
+	uint8_t fanmode;
+	uint16_t rpm_offset;
+	uint8_t duty;
+	uint8_t duty_status;
+	uint16_t rpm_status;
 };
 
 struct icx3_temp_sensors {
-	unsigned char length;
-	unsigned char data[18];
+	uint8_t length;
+	uint8_t data[18];
 };
 
 struct icx3_info {
-	unsigned char length;
-	unsigned char reserved;
-	unsigned char slave_address;
-	unsigned char product_id;
-	unsigned char major_version;
-	unsigned char minor_version;
+	uint8_t length;
+	uint8_t reserved;
+	uint8_t slave_address;
+	uint8_t product_id;
+	uint8_t major_version;
+	uint8_t minor_version;
 };
 
 enum icx3_product_id {
@@ -111,10 +117,14 @@ static char *icx3_temp_sensor_names[] = {
 	"PWR5",
 };
 
-enum icx3_product_id check_for_icx3(char *i2c_dev_path);
-void print_icx3_fans(char *i2c_dev_path);
-void print_icx3_temps(char *i2c_dev_path);
-void get_available_fans(char *i2c_dev_path, char *fans_avail);
-int open_i2c_dev(char *i2c_dev_path);
-void enable_write(int enable, char *i2c_dev_path);
-void set_fan(int fan, char *setting, char *i2c_dev_path);
+int icx3_init(struct card_info *card);
+void print_icx3_fans(struct card_info *card);
+void print_icx3_fans_oneline(struct card_info *card);
+void print_icx3_temps_oneline(struct card_info *card);
+void get_available_fans(char *fans_avail, struct card_info *card);
+void get_fan_status(struct icx3_fan_control *fans, struct card_info *card);
+void get_temp_sensors(float *temps, struct card_info *card);
+void enable_write(int enable, struct card_info *card);
+void set_fan(int fan, char *setting, struct card_info *card);
+
+#endif

makefile

@@ -1,16 +0,0 @@
-.PHONY : clean debug
-
-OBJS = evga-icx.o evga-card.o icx3.o
-LDLIBS = -li2c
-CFLAGS = -MD
-
-evga-icx : $(OBJS)
-
-debug : CFLAGS += -g -O0
-debug : evga-icx
-
-clean :
-	rm evga-icx $(OBJS)
-	rm *.d
-	
--include $(OBJS:.o=.d)

nvidia-dcgm.c

@@ -0,0 +1,127 @@
+#include <stdio.h>
+
+#include "nvidia-dcgm.h"
+
+/* Yeah I know this is duplicated */
+#define HEADER_COLOR_START "\x1b[36m"
+#define HEADER_COLOR_END "\x1b[39m"
+
+dcgmHandle_t 	dcgm_handle = (dcgmHandle_t)NULL;
+dcgmFieldGrp_t 	field_group_id;
+
+unsigned long long clock_reasons = 0;
+
+void init_dcgm()
+{
+	/* Init DCGM */
+	dcgmReturn_t result;
+	result = dcgmInit();
+	if (result != DCGM_ST_OK)
+		printf("Could not init DCGM: %s\n", errorString(result));
+	
+	/* Start embedded */
+	result = dcgmStartEmbedded(DCGM_OPERATION_MODE_MANUAL, &dcgm_handle);
+	if (result != DCGM_ST_OK)
+		printf("Could not start embedded DCGM engine: %s\n", errorString(result));
+	
+	/* Create metric group */
+	int num_metrics = sizeof(metric_readings) / sizeof(metric_readings[0]);
+	unsigned short field_ids[num_metrics + 1];
+	for (int i = 0; i < num_metrics; i++) {
+		field_ids[i] = metric_readings[i].field_id;
+		metric_readings[i].last_reading = 0.0;
+	}
+	field_ids[num_metrics] = DCGM_FI_DEV_CLOCKS_EVENT_REASONS;
+	result = dcgmFieldGroupCreate(dcgm_handle, num_metrics + 1, &field_ids[0], (char *)"evga_icx_fields", &field_group_id);
+	if (result != DCGM_ST_OK)
+		printf("Could not create DCGM metric group: %s\n", errorString(result));
+	
+	/* Start recording */
+	result = dcgmWatchFields(dcgm_handle, DCGM_GROUP_ALL_GPUS , field_group_id, 1000000, 30, 1);
+	if (result != DCGM_ST_OK)
+		printf("Could not record DCGM metrics: %s\n", errorString(result));
+}
+
+void shutdown_dcgm()
+{
+	dcgmStopEmbedded(dcgm_handle);
+	dcgmShutdown();
+}
+
+void print_dcgm(int compact, int overwrite)
+{
+	dcgmReturn_t result;
+	dcgmUpdateAllFields(dcgm_handle, 1);
+	result = dcgmGetLatestValues(dcgm_handle, DCGM_GROUP_ALL_GPUS, field_group_id, &update_field_values, NULL);
+	if (result != DCGM_ST_OK)
+		printf("Could not read DCGM metrics: %s\n", errorString(result));
+	
+	/* Print the float metrics */
+	for (int i = 0; i < sizeof(metric_readings) / sizeof(metric_readings[0]); i++) {
+		if (compact)
+			printf("%s%s%s%s %4.1f",
+				i==0 ? "": " ",
+				HEADER_COLOR_START,
+				metric_readings[i].short_name,
+				HEADER_COLOR_END,
+				metric_readings[i].last_reading * 100);
+		else
+			printf("%s: %.1f%\n", metric_readings[i].long_name, metric_readings[i].last_reading * 100);
+	}
+	if (compact)
+		printf("%%");
+	
+	/* Print clock reasons */
+	int single_reason = 1;
+	if (compact)
+		printf("%s CLK %s", HEADER_COLOR_START, HEADER_COLOR_END);
+	else
+		printf("Clock reasons: ");
+	
+	for (int i = 0; i < (sizeof(dcgm_clock_reason_names) / sizeof(struct clock_reason)); i++) {
+		if (clock_reasons & dcgm_clock_reason_names[i].mask) {
+			if (!single_reason) {
+				if (compact)
+					printf(",");
+				else
+					printf(", ");
+			}	
+			single_reason = 0;
+			
+			if (compact)
+				printf("%s", dcgm_clock_reason_names[i].short_name);
+			else
+				printf("%s", dcgm_clock_reason_names[i].long_name);
+		}
+	}
+	
+	if (single_reason)
+		printf("None");
+	
+	if (!compact)
+		printf(" (0x%llx)", clock_reasons);
+	else
+		printf("%-15s", " ");
+}
+
+int update_field_values(unsigned int gpu_id, dcgmFieldValue_v1 *values, int num_values, void *userdata)
+{
+	for (int j = 0; j < num_values; j++) {
+		if (values[j].fieldType == DCGM_FT_DOUBLE) {
+			/* A double reading so read it into our readings array */
+			for (int i = 0; i < sizeof(metric_readings) / sizeof(metric_readings[0]); i++) {
+				if (metric_readings[i].field_id == values[j].fieldId) {
+					double val = values[j].value.dbl;
+					if (val >= 1)
+						metric_readings[i].last_reading = 0.0;
+					else
+						metric_readings[i].last_reading = values[j].value.dbl;
+				}
+			}
+		} else if (values[j].fieldType == DCGM_FT_INT64) {
+			/* Int is probably the bitmask of our clock reasons */
+			clock_reasons = values[j].value.i64;
+		}
+	}
+	return 0;
+}

nvidia-dcgm.h

@@ -0,0 +1,45 @@
+#include <dcgm_agent.h>
+#include <dcgm_structs.h>
+
+#include "evga-card.h"
+
+struct metric_reading {
+	unsigned short field_id;
+	char *short_name;
+	char *long_name;
+	double last_reading;
+};
+
+static struct metric_reading metric_readings[] =
+{
+	{DCGM_FI_PROF_GR_ENGINE_ACTIVE,			"GFX",		"Graphics Engine Activity"},
+	{DCGM_FI_PROF_SM_ACTIVE,				"SM ACT",	"SM Activity"},
+	{DCGM_FI_PROF_SM_OCCUPANCY,				"OCC",		"SM Occupancy"},
+	{DCGM_FI_PROF_PIPE_FP16_ACTIVE,			"FP16",		"FP16 Engine Activity"},
+	{DCGM_FI_PROF_PIPE_FP32_ACTIVE,			"FP32",		"FP32 Engine Activity"},
+	{DCGM_FI_PROF_PIPE_FP64_ACTIVE,			"FP64",		"FP64 Engine Activity"},
+	{DCGM_FI_PROF_PIPE_INT_ACTIVE,			"INT",		"Integer Activity"},
+	{DCGM_FI_PROF_PIPE_TENSOR_ACTIVE,		"TENS",		"Tensor Activity"},
+	{DCGM_FI_PROF_PIPE_TENSOR_HMMA_ACTIVE,	"HMMA",		"Half Precision Tensor"},
+	{DCGM_FI_PROF_PIPE_TENSOR_IMMA_ACTIVE,	"IMMA",		"Integer Tensor"},
+	{DCGM_FI_PROF_PIPE_TENSOR_DFMA_ACTIVE,	"DFMA",		"Double Precision Tensor"},
+	{DCGM_FI_PROF_DRAM_ACTIVE,				"DRAM",		"Memory BW Utilization"}
+};
+
+static struct clock_reason dcgm_clock_reason_names[] = 
+{
+	{DCGM_CLOCKS_EVENT_REASON_GPU_IDLE,			"Idle", 	"GPU idle"},
+	{DCGM_CLOCKS_EVENT_REASON_CLOCKS_SETTING,	"AppClk",	"Application clocks"},
+	{DCGM_CLOCKS_EVENT_REASON_SW_POWER_CAP,		"Pwr",		"Power cap"},
+	{DCGM_CLOCKS_EVENT_REASON_HW_SLOWDOWN,		"HWSlow",	"Hardware slowdown"},
+	{DCGM_CLOCKS_EVENT_REASON_SYNC_BOOST,		"Sync",		"Sync boost"},
+	{DCGM_CLOCKS_EVENT_REASON_SW_THERMAL,		"SWTherm",	"Software thermal"},
+	{DCGM_CLOCKS_EVENT_REASON_HW_THERMAL,		"HWTherm",	"Hardware thermal"},
+	{DCGM_CLOCKS_EVENT_REASON_HW_POWER_BRAKE,	"HWPower",	"Hardware power brake"},
+	{DCGM_CLOCKS_EVENT_REASON_DISPLAY_CLOCKS,	"DispClk",	"Display clock"}
+};
+
+void init_dcgm();
+void shutdown_dcgm();
+void print_dcgm(int compact, int overwrite);
+int update_field_values(unsigned int gpu_id, dcgmFieldValue_v1 *values, int num_values, void *userdata);

nvidia-sensors.c

@@ -0,0 +1,99 @@
+#include <stdio.h>
+
+#include "nvidia-sensors.h"
+
+void init_nvml()
+{
+	nvmlReturn_t result;
+	result = nvmlInit_v2();
+	if (result != NVML_SUCCESS)
+		printf("Could not init NVML: %s\n", nvmlErrorString(result));
+}
+
+void get_nvml_handle(struct card_info *card)
+{
+	nvmlReturn_t result;
+	result = nvmlDeviceGetHandleByPciBusId_v2(card->pci_id, card->nvml_device);
+	if (result != NVML_SUCCESS) {
+		printf("Failed to get NVML device handle for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
+		card->nvml_device = NULL;
+	}
+}
+
+void print_nvml_clock_reason(int compact, struct card_info *card)
+{
+	unsigned long long reasons = get_nvml_clock_reasons(card);
+	int single_reason = 1;
+		
+	for (int i = 0; i < (sizeof(nvml_clock_reason_names) / sizeof(struct clock_reason)); i++) {
+		if (reasons & nvml_clock_reason_names[i].mask) {
+			if (!single_reason) {
+				if (compact)
+					printf(",");
+				else
+					printf(", ");
+			}	
+			single_reason = 0;
+			
+			if (compact)
+				printf("%s", nvml_clock_reason_names[i].short_name);
+			else
+				printf("%s", nvml_clock_reason_names[i].long_name);
+		}
+	}
+	
+	if (single_reason)
+		printf("None");
+	
+	if (!compact)
+		printf(" (0x%llx)", reasons);
+	else
+		printf("%-15s", " ");
+}
+
+unsigned int get_nvml_temp(struct card_info *card)
+{
+	if (card->nvml_device == NULL)
+		return 0;
+	
+	nvmlTemperature_t temp;
+	temp.version = nvmlTemperature_v1;
+	temp.sensorType = NVML_TEMPERATURE_GPU;
+	nvmlReturn_t result = nvmlDeviceGetTemperatureV(*(nvmlDevice_t*)(card->nvml_device), &temp); 
+	if (result != NVML_SUCCESS) {
+		printf("Failed to get temperature for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
+		return 0;
+	}
+	return temp.temperature;
+}
+
+unsigned long long get_nvml_clock_reasons(struct card_info *card)
+{
+	if (card->nvml_device == NULL)
+		return 0;
+	
+	unsigned long long reasons;
+	nvmlReturn_t result = nvmlDeviceGetCurrentClocksEventReasons(*(nvmlDevice_t*)(card->nvml_device), &reasons) ; 
+	if (result != NVML_SUCCESS) {
+		printf("Failed to get clock reasons for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
+		return 0;
+	}
+	
+	return reasons;
+}
+
+unsigned int get_nvml_mem_util(struct card_info *card)
+{
+	if (card->nvml_device == NULL)
+		return 0;
+	
+	nvmlUtilization_t util;
+	nvmlReturn_t result = nvmlDeviceGetUtilizationRates(*(nvmlDevice_t*)(card->nvml_device), &util);
+	if (result != NVML_SUCCESS) {
+		printf("Failed to get clock reasons for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
+		return 0;
+	}
+	
+	return util.memory;
+}
+

nvidia-sensors.h

@@ -0,0 +1,23 @@
+#include <nvml.h>
+
+#include "evga-card.h"
+
+static struct clock_reason nvml_clock_reason_names[] = 
+{
+	{nvmlClocksEventReasonGpuIdle,						"Idle", 	"GPU idle"},
+	{nvmlClocksEventReasonApplicationsClocksSetting,	"AppClk",	"Application clocks"},
+	{nvmlClocksEventReasonSwPowerCap,					"Pwr",		"Power cap"},
+	{nvmlClocksThrottleReasonHwSlowdown,				"HWSlow",	"Hardware slowdown"},
+	{nvmlClocksEventReasonSyncBoost,					"Sync",		"Sync boost"},
+	{nvmlClocksEventReasonSwThermalSlowdown,			"SWTherm",	"Software thermal"},
+	{nvmlClocksThrottleReasonHwThermalSlowdown,			"HWTherm",	"Hardware thermal"},
+	{nvmlClocksThrottleReasonHwPowerBrakeSlowdown,		"HWPower",	"Hardware power brake"},
+	{nvmlClocksEventReasonDisplayClockSetting,			"DispClk",	"Display clock"}
+};
+
+void init_nvml();
+void get_nvml_handle(struct card_info *card);
+void print_nvml_clock_reason(int compact, struct card_info *card);
+unsigned int get_nvml_temp(struct card_info *card);
+unsigned long long get_nvml_clock_reasons(struct card_info *card);
+unsigned int get_nvml_mem_util(struct card_info *card);

zen3-rapl.c

@@ -0,0 +1,112 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <cpuid.h>
+#include <math.h>
+#include <string.h>
+#include <fcntl.h>
+
+#include "zen3-rapl.h"
+
+#define AMD_STRING "AuthenticAMD"
+#define ZEN_FAMILY 0x17
+#define ZEN3_FAMILY 0x19
+
+#define MEASUREMENT_TIME 0.25
+
+static float energy_unit = 0;
+
+static int msr_file = -1;
+
+static float package_power = 0.0;
+
+static int check_zen()
+{
+    unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0, ext_family;
+    char vendor[13];
+
+    __get_cpuid(0, &eax, &ebx, &ecx, &edx);
+
+    memcpy(vendor, &ebx, 4);
+    memcpy(vendor+4, &edx, 4);
+    memcpy(vendor+8, &ecx, 4);
+    vendor[12] = 0;
+
+    if (strcmp(vendor, AMD_STRING) != 0){
+        return 0;
+    }
+
+    __get_cpuid(1, &eax, &ebx, &ecx, &edx);
+
+    ext_family = ((eax >> 8) & 0xF) + ((eax >> 20) & 0xFF);
+    if (ext_family != ZEN_FAMILY && ext_family != ZEN3_FAMILY){
+        return 0;
+    }
+
+    return 1;
+}
+
+static int read_msr(int file, unsigned int index, unsigned long long *data)
+{
+    if (file < 0)
+        return 0;
+
+    return pread(file, data, sizeof *data, index) == sizeof *data;
+}
+
+static float get_energy_unit()
+{
+    unsigned long long data;
+    // AMD OSRR: page 139 - MSRC001_0299
+    if (!read_msr(msr_file, 0xC0010299, &data))
+        return 0.0;
+
+    return pow(1.0/2.0, (float)((data >> 8) & 0x1F));
+}
+
+static unsigned long get_package_energy()
+{
+    unsigned long long data;
+    // AMD OSRR: page 139 - MSRC001_029B
+    if (!read_msr(msr_file, 0xC001029B, &data))
+        return 0;
+
+    return data;
+}
+
+int init_rapl()
+{
+	/* Check for supported Zen CPU */
+	if (!check_zen())
+		return 0;
+	
+	/* Open MSR file */
+	char *msr_path = "/dev/cpu/0/msr";
+    msr_file = open(msr_path, O_RDONLY);
+	if (msr_file < 0)
+		return 0;
+	
+	/* Energy unit */
+	energy_unit = get_energy_unit();
+    if (energy_unit == 0)
+        return 0;
+	
+	return 1;
+}
+
+float get_rapl_package_power()
+{
+	unsigned long package_eng_b = 0;
+	unsigned long package_eng_a = 0;
+
+    package_eng_b = get_package_energy();
+
+    usleep(MEASUREMENT_TIME*1000000);
+
+    package_eng_a = get_package_energy();
+
+	/* Only update if we computed a good power reading, otherwise return the most recent measurement */
+    if (package_eng_a >= package_eng_b)
+        package_power = (package_eng_a - package_eng_b) * energy_unit / MEASUREMENT_TIME;
+	
+	return package_power;
+}

zen3-rapl.h

@@ -0,0 +1,6 @@
+#include "board-sensors.h"
+
+static struct hwmon_avail_sensor rapl_sensor = {"Package power", -1, NULL, NULL};
+
+int init_rapl();
+float get_rapl_package_power();