3 Commits
Author SHA1 Message Date
moosecrap 812e0f89f3 Documentation 2026-09-30 01:21:46 -07:00
moosecrap fdf90ef790 Fixes when compiling with both NVML and DCGM.
NVML temp code updated (driver version ~555 and up)
2026-09-30 01:18:05 -07:00
moosecrap f6efb12fdb Include guards 2026-09-30 00:40:25 -07:00
10 changed files with 34 additions and 29 deletions
+1 -1
View File
@@ -22,8 +22,8 @@ debug : CFLAGS += -g -Og
debug : evga-icx debug : evga-icx
clean : clean :
rm evga-icx
rm *.o rm *.o
rm *.d rm *.d
rm evga-icx
-include $(OBJS:.o=.d) -include $(OBJS:.o=.d)
+1 -1
View File
@@ -36,7 +36,7 @@ Access to the `/dev/i2c` device files, which means either:
Add the make flag `USE_NVML=1` and the it will also display the main GPU temperature ("GPU1") as reported by the NVIDIA driver. It will also display the performance cap/clock reason and memory controller utilization. This requires the NVIDIA management library (NVML) to be installed. Add the make flag `USE_NVML=1` and the it will also display the main GPU temperature ("GPU1") as reported by the NVIDIA driver. It will also display the performance cap/clock reason and memory controller utilization. This requires the NVIDIA management library (NVML) to be installed.
### VRAM and Hotspot temperature ### VRAM and Hotspot temperature
Run as root and you can also read the VRAM and "hotspot" temperatures. These require access the BAR0 through the `/sys/bus/pci/devices/*/resource0` device files. These sensors are **extremely** undocumented so I can't say anything about their accuracy. If you have a Blackwell (50 series) card you can read all 4 die temp sensors, a 'system' temperature, and every individual VRAM chip temp (add your card in blackwell-temps.c). Run as root and you can also read the VRAM and "hotspot" temperatures. These require access the BAR0 through the `/sys/bus/pci/devices/*/resource0` device files. These sensors are **extremely** undocumented so I can't say anything about their accuracy. If you have a Blackwell (50 series) card you can read all 4 die temp sensors, a 'system' temperature, and every individual VRAM chip temp (add your card in blackwell-temps.c). You will probably require the kernel parameter `iomem=relaxed`.
### Datacenter GPU Manager metrics ### Datacenter GPU Manager metrics
Add the make flag `USE_DCGM=1` and it can display real-time metrics from supported Nvidia GPUs like SM occupancy, integer/fp pipe usage, tensor usage, and DRAM bandwidth utilization. This requires a Quadro/Workstation or a Tesla/Datacenter card, it does not work on GeForce cards. Add the make flag `USE_DCGM=1` and it can display real-time metrics from supported Nvidia GPUs like SM occupancy, integer/fp pipe usage, tensor usage, and DRAM bandwidth utilization. This requires a Quadro/Workstation or a Tesla/Datacenter card, it does not work on GeForce cards.
+1
View File
@@ -3,6 +3,7 @@
#include <dirent.h> #include <dirent.h>
#include <stdint.h> #include <stdint.h>
#include <math.h> #include <math.h>
#include <stdbool.h>
#include "blackwell-sensors.h" #include "blackwell-sensors.h"
+3 -2
View File
@@ -3,7 +3,8 @@
#include "card.h" #include "card.h"
#endif
int find_blackwell_gpus(struct card_info *infos, int num_gpus, int max_gpus); int find_blackwell_gpus(struct card_info *infos, int num_gpus, int max_gpus);
void print_blackwell_info(int compact, int overwrite, struct card_info *gpu); void print_blackwell_info(int compact, int overwrite, struct card_info *gpu);
#endif
+10
View File
@@ -1,3 +1,6 @@
#ifndef CARD_H
#define CARD_H
#define NVIDIA_VEN 0x10DE #define NVIDIA_VEN 0x10DE
#define NVIDIA_RTX3060_DEV 0x2503 #define NVIDIA_RTX3060_DEV 0x2503
@@ -33,6 +36,13 @@ struct card_info {
void *nvml_device; /* Pointer to nvmlDevice_t for use in NVML calls */ void *nvml_device; /* Pointer to nvmlDevice_t for use in NVML calls */
}; };
struct clock_reason {
unsigned long long mask;
char *short_name;
char *long_name;
};
unsigned short read_pci_id(char *device_path, char *field); unsigned short read_pci_id(char *device_path, char *field);
char *read_nvidia_pci_address(char *device_path); char *read_nvidia_pci_address(char *device_path);
void mmap_bar0(struct card_info *card); void mmap_bar0(struct card_info *card);
#endif
+3
View File
@@ -1,3 +1,5 @@
#ifndef GDDR6_H
#define GDDR6_H
#include "evga-card.h" #include "evga-card.h"
struct device_offset { struct device_offset {
@@ -8,3 +10,4 @@ struct device_offset {
float get_vram_temp(struct card_info *card); float get_vram_temp(struct card_info *card);
float get_hotspot_temp(struct card_info *card); float get_hotspot_temp(struct card_info *card);
#endif
+4 -4
View File
@@ -78,8 +78,8 @@ void print_dcgm(int compact, int overwrite)
else else
printf("Clock reasons: "); printf("Clock reasons: ");
for (int i = 0; i < (sizeof(clock_reason_names) / sizeof(struct clock_reason)); i++) { for (int i = 0; i < (sizeof(dcgm_clock_reason_names) / sizeof(struct clock_reason)); i++) {
if (clock_reasons & clock_reason_names[i].mask) { if (clock_reasons & dcgm_clock_reason_names[i].mask) {
if (!single_reason) { if (!single_reason) {
if (compact) if (compact)
printf(","); printf(",");
@@ -89,9 +89,9 @@ void print_dcgm(int compact, int overwrite)
single_reason = 0; single_reason = 0;
if (compact) if (compact)
printf("%s", clock_reason_names[i].short_name); printf("%s", dcgm_clock_reason_names[i].short_name);
else else
printf("%s", clock_reason_names[i].long_name); printf("%s", dcgm_clock_reason_names[i].long_name);
} }
} }
+1 -7
View File
@@ -26,13 +26,7 @@ static struct metric_reading metric_readings[] =
{DCGM_FI_PROF_DRAM_ACTIVE, "DRAM", "Memory BW Utilization"} {DCGM_FI_PROF_DRAM_ACTIVE, "DRAM", "Memory BW Utilization"}
}; };
struct clock_reason { static struct clock_reason dcgm_clock_reason_names[] =
unsigned long long mask;
char *short_name;
char *long_name;
};
static struct clock_reason clock_reason_names[] =
{ {
{DCGM_CLOCKS_EVENT_REASON_GPU_IDLE, "Idle", "GPU idle"}, {DCGM_CLOCKS_EVENT_REASON_GPU_IDLE, "Idle", "GPU idle"},
{DCGM_CLOCKS_EVENT_REASON_CLOCKS_SETTING, "AppClk", "Application clocks"}, {DCGM_CLOCKS_EVENT_REASON_CLOCKS_SETTING, "AppClk", "Application clocks"},
+9 -7
View File
@@ -25,8 +25,8 @@ void print_nvml_clock_reason(int compact, struct card_info *card)
unsigned long long reasons = get_nvml_clock_reasons(card); unsigned long long reasons = get_nvml_clock_reasons(card);
int single_reason = 1; int single_reason = 1;
for (int i = 0; i < (sizeof(clock_reason_names) / sizeof(struct clock_reason)); i++) { for (int i = 0; i < (sizeof(nvml_clock_reason_names) / sizeof(struct clock_reason)); i++) {
if (reasons & clock_reason_names[i].mask) { if (reasons & nvml_clock_reason_names[i].mask) {
if (!single_reason) { if (!single_reason) {
if (compact) if (compact)
printf(","); printf(",");
@@ -36,9 +36,9 @@ void print_nvml_clock_reason(int compact, struct card_info *card)
single_reason = 0; single_reason = 0;
if (compact) if (compact)
printf("%s", clock_reason_names[i].short_name); printf("%s", nvml_clock_reason_names[i].short_name);
else else
printf("%s", clock_reason_names[i].long_name); printf("%s", nvml_clock_reason_names[i].long_name);
} }
} }
@@ -56,13 +56,15 @@ unsigned int get_nvml_temp(struct card_info *card)
if (card->nvml_device == NULL) if (card->nvml_device == NULL)
return 0; return 0;
unsigned int temp; nvmlTemperature_t temp;
nvmlReturn_t result = nvmlDeviceGetTemperature(*(nvmlDevice_t*)(card->nvml_device), NVML_TEMPERATURE_GPU, &temp); temp.version = nvmlTemperature_v1;
temp.sensorType = NVML_TEMPERATURE_GPU;
nvmlReturn_t result = nvmlDeviceGetTemperatureV(*(nvmlDevice_t*)(card->nvml_device), &temp);
if (result != NVML_SUCCESS) { if (result != NVML_SUCCESS) {
printf("Failed to get temperature for card at %s: %s\n", card->pci_id, nvmlErrorString(result)); printf("Failed to get temperature for card at %s: %s\n", card->pci_id, nvmlErrorString(result));
return 0; return 0;
} }
return temp; return temp.temperature;
} }
unsigned long long get_nvml_clock_reasons(struct card_info *card) unsigned long long get_nvml_clock_reasons(struct card_info *card)
+1 -7
View File
@@ -2,13 +2,7 @@
#include "evga-card.h" #include "evga-card.h"
struct clock_reason { static struct clock_reason nvml_clock_reason_names[] =
unsigned long long mask;
char *short_name;
char *long_name;
};
static struct clock_reason clock_reason_names[] =
{ {
{nvmlClocksEventReasonGpuIdle, "Idle", "GPU idle"}, {nvmlClocksEventReasonGpuIdle, "Idle", "GPU idle"},
{nvmlClocksEventReasonApplicationsClocksSetting, "AppClk", "Application clocks"}, {nvmlClocksEventReasonApplicationsClocksSetting, "AppClk", "Application clocks"},