Added NVML memory controller load percentage.

GPU temp sensor and clock reasons.

LICENSE

@@ -0,0 +1,8 @@
+MIT No Attribution
+
+Copyright 2025 admin@long-cat.net
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

Makefile

@@ -4,13 +4,26 @@ OBJS = evga-icx.o evga-card.o icx3.o
 LDLIBS = -li2c
 CFLAGS = -MD
 
+ifdef USE_NVML
+LDLIBS += -lnvidia-ml
+CFLAGS += -DUSE_NVML
+OBJS += nvidia-sensors.o
+endif
+
+ifdef USE_LIBPCI
+LDLIBS += -lpci
+CFLAGS += -DUSE_LIBPCI
+OBJS += gddr6.o
+endif
+
 evga-icx : $(OBJS)
 
 debug : CFLAGS += -g -O0
 debug : evga-icx
 
 clean :
-	rm evga-icx $(OBJS)
+	rm evga-icx
+	rm *.o
 	rm *.d
 	
 -include $(OBJS:.o=.d)

README.md

@@ -20,10 +20,20 @@ Access to the `/dev/i2c` device files, which means either:
 
 ## Dependencies
 * libi2c-dev
+* libnvidia-ml-dev (if building with `USE_NVML=1`)
+* libpci-dev (if building with `USE_LIBPCI=1`)
 
 ## Building
 `make`
 
+## Optional features
+
+### NVML support
+Add the make flag `USE_NVML=1` and the it will also display the main GPU temperature ("GPU1") as reported by the NVIDIA driver.  It will also display the performance cap/clock reason and memory controller utilization.  This requires the NVIDIA management library (NVML) to be installed.
+
+### VRAM and Hotspot temperature
+Add the make flag `USE_LIBPCI=1` and you can also read the VRAM and "hotspot" temperatures.  These require direct memory access to the PCI device so you must run as root and also enable the kernel parameter `iomem=relaxed`.  These sensors are **extremely** undocumented so I can't say anything about their accuracy.
+
 ## Usage
 Note that when controlling fans directly through iCX3 they will fall offline from the Nvidia driver and show as 0 RPM until you return them to automatic mode.
 
@@ -40,6 +50,8 @@ Available options:
 --sensors    : Print sensor readings even if setting a fan speed
 --compact    : Print sensor reading in a compact one-line per card format
 --watch N    : Keep printing output every N seconds
+--overwrite  : Overwrite previously displayed info with --watch and --compact instead of continuously logging new lines
+--color      : Print headers in color in --compact mode for better readability
 ```
 
 ### Examples:
@@ -47,19 +59,30 @@ Read sensors:
 ```text
 $ ./evga-icx
 #0: EVGA GeForce RTX 3090 FTW3 Ultra v2 (/dev/i2c-3) @ c:00.0
-Fan 0: 1934 RPM (64/0%, Auto)
+Fan 0: 1751 RPM (58/0%, Auto)
-Fan 1: 1930 RPM (64/0%, Auto)
+Fan 1: 1730 RPM (57/0%, Auto)
-Fan 2: 1944 RPM (64/0%, Offset)
+Fan 2: 1712 RPM (57/0%, Offset)
 Ext. fan: 0 RPM (0/0%, Offset)
-GPU2: +64.1°C
+GPU1: +65°C
-MEM1: +62.2°C
+GPU2: +57.8°C
-MEM2: +58.2°C
+VRAM: +74°C
-MEM3: +62.2°C
+MEM1: +56.1°C
-PWR1: +54.3°C
+MEM2: +53.5°C
-PWR2: +59.4°C
+MEM3: +55.5°C
-PWR3: +66.0°C
+PWR1: +48.2°C
-PWR4: +64.3°C
+PWR2: +53.2°C
-PWR5: +57.6°C
+PWR3: +59.6°C
+PWR4: +58.0°C
+PWR5: +51.1°C
+HotSpot: +75°C
+Mem util: 43%
+Clock reasons: Power cap (0x4)
+```
+
+Compact one-line mode:
+```text
+$ ./evga-icx --compact
+#0 FAN  59  59  58   0% GPU  66  60 MEM  74  58  55  58 PWR  49  55  61  60  53 HOT  77°C  MEM  42% CLK Pwr
 ```
 
 Set external fan to follow Nvidia driver controlled speed with a -500 RPM offset:

evga-card.c

@@ -67,6 +67,7 @@ int find_evga_gpu_i2cs(struct card_info *infos, int max_gpus)
 					/* Write our card info into the provided struct array */
 					infos[num_gpus].card_name = evga_pci_ids[i].card_name;
 					infos[num_gpus].pci_id = pci_addr;
+					infos[num_gpus].pci_device_id = pci_device;
 					infos[num_gpus].i2c_dev_path = calloc(strlen(dev_file) + 1, sizeof(char));
 					strcpy(infos[num_gpus].i2c_dev_path, dev_file);
 					num_gpus++;

evga-card.h

@@ -84,11 +84,13 @@
 #define EVGA_RTX3090TI_FTW3_ULTRA_GAMING_SUB_DEV        0x4985
 
 struct card_info {
-	char *card_name;
+	char *card_name;					/* The 'nice' name of the card */
-	char *pci_id;
+	char *pci_id;						/* PCI bus address in domain:bus:device.function format. May be shortened (e.g. c:00.0) */
-	char *i2c_dev_path;
+	unsigned short pci_device_id;		/* The device ID of the card, i.e. corresponds to the NVIDIA model number */
-	int i2c_fd;
+	char *i2c_dev_path;					/* Path to the i2c device file */
-	int product_id;
+	int i2c_fd;							/* File descriptor for the i2c device file, for re-use */
+	int product_id;						/* EVGA internal product ID, as reported by the iCX3 controller */
+	unsigned int bar0;					/* Address of the card's PCI base address register */
 };
 
 struct gpu_pci_info {

evga-icx.c

@@ -5,10 +5,23 @@
 #include <fcntl.h>
 #include <stdlib.h>
 
+#ifdef USE_NVML
+#include "nvidia-sensors.h"
+#endif
+
+#ifdef USE_LIBPCI
+#include "gddr6.h"
+#endif
+
 #include "icx3.h"
 #include "evga-card.h"
 
 #define MAX_GPUS 16
+#define HEADER_COLOR_START "\x1b[36m"
+#define HEADER_COLOR_END "\x1b[39m"
+
+char *header_start = "";
+char *header_end = "";
 
 static const char helpstring[] = "Available options:\n"
                                  "--gpu N      : Control only GPU N instead of all supported cards\n"
@@ -21,7 +34,9 @@ static const char helpstring[] = "Available options:\n"
 								 "--reset      : Reset all fans to their default mode\n"
 								 "--sensors    : Print sensor readings even if setting a fan speed \n"
 								 "--compact    : Print sensor reading in a compact one-line per card format\n"
-								 "--watch N    : Keep printing output every N seconds\n";
+								 "--watch N    : Keep printing output every N seconds\n"
+								 "--overwrite  : Overwrite previously displayed info with --watch and --compact instead of continuously logging\n"
+								 "--color      : Print headers in color in --compact mode for better readability\n";
 								 
 void print_gpu_info(int gpu_num, struct card_info gpus[], int compact);
 
@@ -32,7 +47,8 @@ int main (int argc, char **argv)
 	int print_info = 0;
 	int compact = 0;
 	int gpu_num = -1;    /* Card to control */
-	int watch = -1;
+	int overwrite = 0;
+	unsigned int watch = 0;
 	char *fan_speed[ICX3_MAX_FANS] = {NULL}; 
 	
 	/* Input parsing */
@@ -79,6 +95,11 @@ int main (int argc, char **argv)
 				printf(helpstring);
 				return -1;
 			}
+		} else if (strcmp(argv[i], "--overwrite") == 0) {
+			overwrite = 1;
+		} else if (strcmp(argv[i], "--color") == 0) {
+			header_start = HEADER_COLOR_START;
+			header_end = HEADER_COLOR_END;
 		} else {
 			printf(helpstring);
 			return 0;
@@ -94,6 +115,10 @@ int main (int argc, char **argv)
 		}
 	}
 	
+	/* Don't use overwrite mode unless set to compact (we can't tell how many lines the output will be per GPU) */
+	if (overwrite && !compact)
+		overwrite = 0;
+	
 	gpu_count = find_evga_gpu_i2cs(gpus, MAX_GPUS);
 
 	if (gpu_count == -1) {
@@ -125,34 +150,119 @@ int main (int argc, char **argv)
 		}
 	}
 	
+	/* NVML init */
+#ifdef USE_NVML
+    init_nvml();
+#endif
+
+	/* PCI init for VRAM/hotspot temps */
+#ifdef USE_LIBPCI
+	for (int i = 0; i < gpu_count; i++)
+		init_gddr6(&gpus[i]);
+#endif
+	
 	/* print sensor info */
-print:
 	if (print_info) {
-		if (gpu_num == -1) {
+		do {
-			for (int i = 0; i < gpu_count; i++){
+			printf("\x1b[K");	/* Clear current console line (really just for overwrite mode) */
-				print_gpu_info(i, gpus, compact);
+			
+			if (gpu_num == -1) {
+				/* No GPU specified on command line, loop over all supported GPUs */
+				for (int i = 0; i < gpu_count; i++){
+					if (i > 0)
+						printf("\n");
+					print_gpu_info(i, &gpus[i], compact);
+				}		
+			} else if (gpu_num <= gpu_count - 1) {
+				print_gpu_info(gpu_num, &gpus[gpu_num], compact);
 			}
-		} else if (gpu_num <= gpu_count - 1) {
+
-			print_gpu_info(gpu_num, gpus, compact);
+			if (!overwrite)
-		}
+				printf("\n"); /* Print line break at the end for continuous output */
-	}
+			
-	if (watch > 0) {
+			if (overwrite && compact) {
-		sleep(watch);
+				printf("\x1b[1G");						/* Move cursor back to column 1 */
-		goto print;
+				if (gpu_count > 1)
+					printf("\x1b[%dA", gpu_count-1);	/* Move cursor back up to the top of gpu list */
+			}
+			
+			fflush(stdout);
+			sleep(watch);
+		} while (watch > 0);
 	}
+	
+#ifdef USE_NVML
+	nvmlShutdown();
+#endif
 }
 
-void print_gpu_info(int gpu_num, struct card_info gpus[], int compact) {
+void print_gpu_info(int gpu_num, struct card_info *gpu, int compact) {
 	if (compact) {
-		printf("#%d ", gpu_num);
+		/* One line per GPU */
-		print_icx3_fans_oneline(&gpus[gpu_num]);
+		printf("%s#%d FAN%s", header_start, gpu_num, header_end);
-		print_icx3_temps_oneline(&gpus[gpu_num]);
+		print_icx3_fans_oneline(gpu);
-		printf("\n");
+		printf("%s GPU%s", header_start, header_end);
+		
+#ifdef USE_NVML
+		printf(" %3d", get_nvml_temp(gpu));
+#endif
+		
+		float icx_temp_sensors[ICX3_NUM_TEMP_SENSORS] = {};
+		get_temp_sensors(icx_temp_sensors, gpu);
+		for (int i = 0; i < ICX3_NUM_TEMP_SENSORS; i++) {
+			if (i > 0 && strncmp(icx3_temp_sensor_names[i], icx3_temp_sensor_names[i-1], 3))
+				printf("%s %.3s%s", header_start, icx3_temp_sensor_names[i], header_end);
+#ifdef USE_LIBPCI
+			if (strncmp(icx3_temp_sensor_names[i], "MEM1", 4) == 0)
+				printf(" %3.0f", get_vram_temp(gpu));			/* Print the VRAM temp before the rest of the memory sensors */
+#endif
+			printf(" %3.0f", icx_temp_sensors[i]);
+		}
+		
+#ifdef USE_LIBPCI
+		printf("%s HOT%s %3.0f", header_start, header_end, get_hotspot_temp(gpu));
+#endif
+		
+		printf("°C ");
+		
+#ifdef USE_NVML
+		printf("%s MEM %s", header_start, header_end);
+		printf("%3d%%", get_nvml_mem_util(gpu));
+		printf("%s CLK %s", header_start, header_end);
+		print_nvml_clock_reason(1, gpu);
+#endif
+
+
 	} else {
-		printf("#%d: %s (%s) @ %s\n", gpu_num, gpus[gpu_num].card_name, gpus[gpu_num].i2c_dev_path, gpus[gpu_num].pci_id);
+		/* One line per GPU sensor */
-		print_icx3_fans(&gpus[gpu_num]);
+		printf("#%d: %s (%s) @ %s\n", gpu_num, gpu->card_name, gpu->i2c_dev_path, gpu->pci_id);
-		print_icx3_temps(&gpus[gpu_num]);
+		print_icx3_fans(gpu);
+		
+#ifdef USE_NVML
+		printf("GPU1: %+d°C\n", get_nvml_temp(gpu));
+#endif
+		float icx_temp_sensors[ICX3_NUM_TEMP_SENSORS] = {};
+		get_temp_sensors(icx_temp_sensors, gpu);
+		for (int i = 0; i < ICX3_NUM_TEMP_SENSORS; i++) {
+#ifdef USE_LIBPCI
+			if (strncmp(icx3_temp_sensor_names[i], "MEM1", 4) == 0)
+				printf("VRAM: +%.0f°C\n", get_vram_temp(gpu));			/* Print the VRAM temp before the rest of the memory sensors */
+#endif
+			printf("%s: %+.1f°C\n",
+			icx3_temp_sensor_names[i],
+			icx_temp_sensors[i]);
+		}
+		
+#ifdef USE_LIBPCI
+		printf("HotSpot: +%.0f°C\n", get_hotspot_temp(gpu));
+#endif
+
+#ifdef USE_NVML
+		printf("Mem util: %d%%\n", get_nvml_mem_util(gpu));
+		printf("Clock reasons: ");
+		print_nvml_clock_reason(0, gpu);
 		printf("\n");
+#endif
 	}
 	
 }

gddr6.c

@@ -0,0 +1,105 @@
+#include <string.h>
+#include <stdio.h>
+#include <pci/pci.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#define PG_SZ sysconf(_SC_PAGE_SIZE)
+
+#include "gddr6.h"
+
+void init_gddr6(struct card_info *card)
+{
+	/* Parse the address of the card to get the PCI info */
+	char pci_address[] = "00000000:00:00.0";
+	int len = strlen(card->pci_id);
+	strcpy(&pci_address[sizeof(pci_address) - len - 1], card->pci_id);
+	
+	int domain = 0;
+	int bus = 0;
+	int dev = 0;
+	int func = 0;
+	
+	sscanf(pci_address, "%x:%x:%x.%x", &domain, &bus, &dev, &func);
+	
+	struct pci_access *pacc = NULL;
+    struct pci_dev *pci_dev = NULL;
+
+	pacc = pci_alloc();
+	pci_init(pacc);
+
+	pci_dev = pci_get_dev(pacc, domain, bus, dev, func);
+
+	pci_fill_info(pci_dev, PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS);
+
+	card->bar0 = (pci_dev->base_addr[0] & 0xFFFFFFFF);
+
+	pci_cleanup(pacc);
+}
+
+float get_vram_temp(struct card_info *card)
+{
+	int fd;
+	float temp = 0.0;
+	
+	if ((fd = open("/dev/mem", O_RDWR | O_SYNC)) == -1)
+    {
+        printf("Can't read memory. If you are root, enable kernel parameter iomem=relaxed\n");
+		return 0;
+    }
+	
+	for (int i = 0; i < sizeof(device_offset_info) / sizeof(struct device_offset); i++) {
+		if (card->pci_device_id == device_offset_info[i].device_id){
+			unsigned int phys_addr = (card->bar0 + device_offset_info[i].vram_offset);
+			unsigned int base_offset = phys_addr & ~(PG_SZ-1);
+			void *map_base = mmap(0, PG_SZ, PROT_READ, MAP_SHARED, fd, base_offset);
+			if(map_base == (void *) -1)
+			{
+				if (fd != -1)
+					close(fd);
+				printf("Can't read memory for VRAM temperature. If you are root, enable kernel parameter iomem=relaxed\n");
+			}
+			void *virt_addr = (char *) map_base + (phys_addr - base_offset);
+			int read_result = *((unsigned int *) virt_addr);
+			temp = ((read_result & 0x00000fff) / 0x20);
+			munmap(map_base, PG_SZ);
+		}
+	}
+	
+	close(fd);
+	return temp;
+}
+
+float get_hotspot_temp(struct card_info *card)
+{
+	int fd;
+	float temp = 0.0;
+	
+	if ((fd = open("/dev/mem", O_RDWR | O_SYNC)) == -1)
+    {
+        printf("Can't read memory. If you are root, enable kernel parameter iomem=relaxed\n");
+		return 0;
+    }
+	
+	for (int i = 0; i < sizeof(device_offset_info) / sizeof(struct device_offset); i++) {
+		if (card->pci_device_id == device_offset_info[i].device_id){
+			unsigned int phys_addr = (card->bar0 + device_offset_info[i].hotspot_offset);
+			unsigned int base_offset = phys_addr & ~(PG_SZ-1);
+			void *map_base = mmap(0, PG_SZ, PROT_READ, MAP_SHARED, fd, base_offset);
+			if(map_base == (void *) -1)
+			{
+				if (fd != -1)
+					close(fd);
+				printf("Can't read memory for hotspot. If you are root, enable kernel parameter iomem=relaxed\n");
+			}
+			void *virt_addr = (char *) map_base + (phys_addr - base_offset);
+			int read_result = *((unsigned int *) virt_addr);
+			temp = (read_result >> 8) & 0xff;
+			munmap(map_base, PG_SZ);
+		}
+	}
+	
+	close(fd);
+	return temp;
+}

gddr6.h

@@ -0,0 +1,26 @@
+#include "evga-card.h"
+
+struct device_offset {
+	unsigned short device_id;
+	int vram_offset;
+	int hotspot_offset;
+};
+
+static struct device_offset device_offset_info[] = 
+{
+	{.device_id = NVIDIA_RTX3090TI_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3090 Ti */
+	{.device_id = NVIDIA_RTX3090_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3090 */
+	{.device_id = NVIDIA_RTX3080TI_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 Ti */
+	{.device_id = NVIDIA_RTX3080_12G_LHR_DEV,	.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 12G LHR */
+	{.device_id = NVIDIA_RTX3080_LHR_DEV,		.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 LHR */
+	{.device_id = NVIDIA_RTX3080_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 */
+	{.device_id = NVIDIA_RTX3080_DEV,			.vram_offset = 0x0000E2A8,	.hotspot_offset = 0x0002046c},	/* RTX 3080 */
+	{.device_id = NVIDIA_RTX3070TI_GA102_DEV,	.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 Ti GA102 TODO:check */
+	{.device_id = NVIDIA_RTX3070TI_DEV,			.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 Ti TODO:check */
+	{.device_id = NVIDIA_RTX3070_LHR_DEV,		.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 */
+	{.device_id = NVIDIA_RTX3070_DEV,			.vram_offset = 0x0000EE50,	.hotspot_offset = 0x0002046c},	/* RTX 3070 LHR */
+};
+
+void init_gddr6(struct card_info *card);
+float get_vram_temp(struct card_info *card);
+float get_hotspot_temp(struct card_info *card);

icx3.c

@@ -62,40 +62,12 @@ void print_icx3_fans_oneline(struct card_info *card)
 	struct icx3_fan_control fans[ICX3_MAX_FANS]; 
 	get_fan_status(fans, card);
 	
-	printf("FAN");
 	for (int i=0; i < ICX3_MAX_FANS; i++) {		
 		printf(" %3d", fans[i].duty_status);
 	}
 	printf("%%");
 }
 
-void print_icx3_temps(struct card_info *card)
-{
-	float temps[ICX3_NUM_TEMP_SENSORS];
-	
-	get_temp_sensors(temps, card);
-	
-	for (int i=0; i<ICX3_NUM_TEMP_SENSORS; i++) {
-		printf("%s: %+.1f°C\n",
-		icx3_temp_sensor_names[i],
-		temps[i]);
-	}
-}
-
-void print_icx3_temps_oneline(struct card_info *card)
-{
-	float temps[ICX3_NUM_TEMP_SENSORS];
-	
-	get_temp_sensors(temps, card);
-	
-	for (int i=0; i<ICX3_NUM_TEMP_SENSORS; i++) {
-		if (i == 0 || strncmp(icx3_temp_sensor_names[i], icx3_temp_sensor_names[i-1], 3))
-			printf(" %.3s", icx3_temp_sensor_names[i]);
-		printf(" %3.0f", temps[i]);
-	}
-	printf("°C");
-}
-
 void get_available_fans(char *fans_avail, struct card_info *card)
 {
 	int product_id = card->product_id;

icx3.h

@@ -119,7 +119,6 @@ static char *icx3_temp_sensor_names[] = {
 int icx3_init(struct card_info *card);
 void print_icx3_fans(struct card_info *card);
 void print_icx3_fans_oneline(struct card_info *card);
-void print_icx3_temps(struct card_info *card);
 void print_icx3_temps_oneline(struct card_info *card);
 void get_available_fans(char *fans_avail, struct card_info *card);
 void get_fan_status(struct icx3_fan_control *fans, struct card_info *card);

nvidia-sensors.c

@@ -0,0 +1,99 @@
+#include <stdio.h>
+
+#include "nvidia-sensors.h"
+
+void init_nvml()
+{
+	nvmlReturn_t result;
+	result = nvmlInit_v2();
+	if (result != NVML_SUCCESS)
+		printf("Could not init NVML: %s\n", nvmlErrorString(result));
+}
+
+int get_nvml_handle(nvmlDevice_t *device, struct card_info *card)
+{
+	nvmlReturn_t result;
+	result = nvmlDeviceGetHandleByPciBusId_v2(card->pci_id, device);
+	if (result != NVML_SUCCESS) {
+		printf("Failed to get device handle for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
+		return 0;
+	}
+	return 1;
+}
+
+void print_nvml_clock_reason(int compact, struct card_info *card)
+{
+	unsigned long long reasons = get_nvml_clock_reasons(card);
+	int single_reason = 1;
+		
+	for (int i = 0; i < (sizeof(clock_reason_names) / sizeof(struct clock_reason)); i++) {
+		if (reasons & clock_reason_names[i].mask) {
+			if (!single_reason) {
+				if (compact)
+					printf(",");
+				else
+					printf(", ");
+			}	
+			single_reason = 0;
+			
+			if (compact)
+				printf("%s", clock_reason_names[i].short_name);
+			else
+				printf("%s", clock_reason_names[i].long_name);
+		}
+	}
+	
+	if (single_reason)
+		printf("None");
+	
+	if (!compact)
+		printf(" (0x%llx)", reasons);
+}
+
+unsigned int get_nvml_temp(struct card_info *card)
+{
+	nvmlDevice_t nvml_device;
+	if (!get_nvml_handle(&nvml_device, card))
+		return 0;
+	
+	unsigned int temp;
+	nvmlReturn_t result = nvmlDeviceGetTemperature(nvml_device, NVML_TEMPERATURE_GPU, &temp); 
+	if (result != NVML_SUCCESS) {
+		printf("Failed to get temperature for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
+		return 0;
+	}
+	return temp;
+}
+
+unsigned long long get_nvml_clock_reasons(struct card_info *card)
+{
+	nvmlDevice_t nvml_device;
+	if (!get_nvml_handle(&nvml_device, card))
+		return 0;
+	
+	unsigned long long reasons;
+	nvmlReturn_t result = nvmlDeviceGetCurrentClocksEventReasons(nvml_device, &reasons) ; 
+	if (result != NVML_SUCCESS) {
+		printf("Failed to get clock reasons for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
+		return 0;
+	}
+	
+	return reasons;
+}
+
+unsigned int get_nvml_mem_util(struct card_info *card)
+{
+	nvmlDevice_t nvml_device;
+	if (!get_nvml_handle(&nvml_device, card))
+		return 0;
+	
+	nvmlUtilization_t util;
+	nvmlReturn_t result = nvmlDeviceGetUtilizationRates(nvml_device, &util);
+	if (result != NVML_SUCCESS) {
+		printf("Failed to get clock reasons for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
+		return 0;
+	}
+	
+	return util.memory;
+}
+

nvidia-sensors.h

@@ -0,0 +1,29 @@
+#include <nvml.h>
+
+#include "evga-card.h"
+
+struct clock_reason {
+	unsigned long long mask;
+	char *short_name;
+	char *long_name;
+};
+
+static struct clock_reason clock_reason_names[] = 
+{
+	{nvmlClocksEventReasonGpuIdle,						"Idle", 	"GPU idle"},
+	{nvmlClocksEventReasonApplicationsClocksSetting,	"AppClk",	"Application clocks"},
+	{nvmlClocksEventReasonSwPowerCap,					"Pwr",		"Power cap"},
+	{nvmlClocksThrottleReasonHwSlowdown,				"HWSlow",	"Hardware slowdown"},
+	{nvmlClocksEventReasonSyncBoost,					"Sync",		"Sync boost"},
+	{nvmlClocksEventReasonSwThermalSlowdown,			"SWTherm",	"Software thermal"},
+	{nvmlClocksThrottleReasonHwThermalSlowdown,			"HWTherm",	"Hardware thermal"},
+	{nvmlClocksThrottleReasonHwPowerBrakeSlowdown,		"HWPower",	"Hardware power brake"},
+	{nvmlClocksEventReasonDisplayClockSetting,			"DispClk",	"Display clock"}
+};
+
+void init_nvml();
+int get_nvml_handle(nvmlDevice_t *device, struct card_info *card);
+void print_nvml_clock_reason(int compact, struct card_info *card);
+unsigned int get_nvml_temp(struct card_info *card);
+unsigned long long get_nvml_clock_reasons(struct card_info *card);
+unsigned int get_nvml_mem_util(struct card_info *card);