From 33882d54552dba2fa863471e4bdc6d2e999f2bb1 Mon Sep 17 00:00:00 2001 From: moosecrap Date: Mon, 2 Nov 2026 17:55:03 -0800 Subject: [PATCH] Initial commit --- README.md | 3 ++ ecc2ganglia | 17 +++++++ gpu2ganglia | 109 ++++++++++++++++++++++++++++++++++++++++++++ hw2ganglia | 40 ++++++++++++++++ infiniband2ganglia | 21 +++++++++ ipmisensors2ganglia | 56 +++++++++++++++++++++++ psi2ganglia | 25 ++++++++++ sensors2ganglia | 103 +++++++++++++++++++++++++++++++++++++++++ slurmnode2ganglia | 60 ++++++++++++++++++++++++ slurmqueue2ganglia | 44 ++++++++++++++++++ smart2ganglia | 15 ++++++ systemd2ganglia | 5 ++ ups2ganglia | 45 ++++++++++++++++++ 13 files changed, 543 insertions(+) create mode 100644 README.md create mode 100644 ecc2ganglia create mode 100755 gpu2ganglia create mode 100644 hw2ganglia create mode 100644 infiniband2ganglia create mode 100644 ipmisensors2ganglia create mode 100755 psi2ganglia create mode 100755 sensors2ganglia create mode 100644 slurmnode2ganglia create mode 100644 slurmqueue2ganglia create mode 100755 smart2ganglia create mode 100644 systemd2ganglia create mode 100755 ups2ganglia diff --git a/README.md b/README.md new file mode 100644 index 0000000..7817d72 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +A collection of various scripts that use `gmetric` to send statistics to [Ganglia](https://github.com/ganglia/). + +You may find it useful to combine these into one script for the metrics you would like \ No newline at end of file diff --git a/ecc2ganglia b/ecc2ganglia new file mode 100644 index 0000000..1c5a6e2 --- /dev/null +++ b/ecc2ganglia @@ -0,0 +1,17 @@ +#!/bin/bash + +# ECC memory error count +# only if we have ECC memory and EDAC is enabled +if [[ -e /sys/devices/system/edac ]]; then + # loop over memory controllers + correctable_errors=0 + uncorrectable_errors=0 + for mc in /sys/devices/system/edac/mc/mc*; do + correctable_errors=$(($correctable_errors + $(cat $mc/ce_count))) + uncorrectable_errors=$(($uncorrectable_errors + $(cat $mc/ue_count))) + done + + gmetric --name="ecc_correctable_errors" --title="ECC correctable errors" --value="$correctable_errors" --type="uint32" --group="memory" --conf=$conf + gmetric --name="ecc_uncorrectable_errors" --title="ECC uncorrectable errors" --value="$uncorrectable_errors" --type="uint32" --group="memory" --conf=$conf + +fi diff --git a/gpu2ganglia b/gpu2ganglia new file mode 100755 index 0000000..6862ca2 --- /dev/null +++ b/gpu2ganglia @@ -0,0 +1,109 @@ +#!/bin/bash + +# ganglia_name | nvidia-smi query | type | units | aggregation (sum|avg|none) | title +metrics=( + "gpu_driver_version|driver_version|string||none|GPU driver version" + "gpu_name|gpu_name|string||none|GPU name" + "gpu_vbios_version|vbios_version|string||none|GPU VBIOS version" + "gpu_pcie_link_gen_current|pcie.link.gen.gpucurrent|uint8||none|GPU current PCIe link generation" + "gpu_pcie_link_gen_max|pcie.link.gen.max|uint8||none|GPU maximum PCIe link generation" + "gpu_pcie_link_width_current|pcie.link.width.current|uint8||none|GPU current PCIe link width" + "gpu_pcie_link_width_max|pcie.link.width.max|uint8||none|GPU maximum PCIe link width" + "gpu_memory_total|memory.total|uint32|MB|sum|GPU total memory" + "gpu_memory_reserved|memory.reserved|uint32|MB|sum|GPU reserved memory" + "gpu_memory_used|memory.used|uint32|MB|sum|GPU used memory" + "gpu_memory_free|memory.free|uint32|MB|sum|GPU free memory" + "gpu_fan_speed|fan.speed|float|%|avg|GPU fan speed" + "gpu_utilization|utilization.gpu|float|%|avg|GPU utilization" + "gpu_memory_utilization|utilization.memory|float|%|avg|GPU memory utilization" + "gpu_temperature|temperature.gpu|float|C|avg|GPU temperature" + "gpu_power_draw|power.draw|float|W|sum|GPU power draw" + "gpu_power_limit|power.limit|float|W|sum|GPU power limit" + "gpu_clocks_current_graphics|clocks.current.graphics|float|MHz|avg|GPU graphics clock" + "gpu_clocks_current_sm|clocks.current.sm|float|MHz|avg|GPU streaming multiprocessor clock" + "gpu_clocks_current_memory|clocks.current.memory|float|MHz|avg|GPU memory clock" + "gpu_clocks_current_video|clocks.current.video|float|MHz|avg|GPU video clock" +) + +# Build comma-separated query +smi_query="" +for m in "${metrics[@]}"; do + IFS='|' read -r _ query _ _ _ _ <<< "$m" + smi_query+="$query," +done +smi_query=${smi_query%,} + +raw=$(nvidia-smi --format=csv,noheader,nounits --query-gpu="$smi_query" 2>/dev/null) + +# Get GPU count (nvidia-smi prints it once per GPU, so take first line) +gpu_count=$(nvidia-smi --format=csv,noheader,nounits --query-gpu=count 2>/dev/null | head -1) + +# Accumulators +declare -a agg_sum +declare -a agg_count +for i in "${!metrics[@]}"; do + agg_sum[$i]=0 + agg_count[$i]=0 +done + +metric() { + local name="$1" value="$2" type="$3" units="$4" title="$5" + [[ -z "$value" ]] && return + local args=(--name="$name" --title="$title" --value="$value" --type="$type" --group=gpu) + [[ -n "$units" ]] && args+=(--units="$units") + gmetric "${args[@]}" +} + +gpu_idx=0 +while IFS= read -r line; do + [[ -z "$line" ]] && continue + IFS=',' read -ra values <<< "$line" + + mi=0 + for m in "${metrics[@]}"; do + IFS='|' read -r ganglia_name _ type units agg title <<< "$m" + value="${values[$mi]}" + # nvidia-smi CSV puts a space after each comma; trim leading whitespace + value="${value#"${value%%[![:space:]]*}"}" + mi=$((mi + 1)) + [[ -z "$value" ]] && continue + + # Single GPU: emit non-indexed (preserves history) + # Multiple GPUs: emit indexed per-GPU + if [[ $gpu_count -eq 1 ]]; then + metric "$ganglia_name" "$value" "$type" "$units" "$title" + else + base="${ganglia_name#gpu_}" + # Replace 'GPU' with 'GPU N' in title for per-GPU metrics + gpu_title="${title/GPU/GPU $gpu_idx}" + metric "gpu${gpu_idx}_${base}" "$value" "$type" "$units" "$gpu_title" + fi + + # Accumulate for aggregate + if [[ "$agg" == "sum" || "$agg" == "avg" ]]; then + idx=$((mi - 1)) + agg_sum[$idx]=$(awk "BEGIN {print ${agg_sum[$idx]} + $value}") + agg_count[$idx]=$((agg_count[$idx] + 1)) + fi + done + + gpu_idx=$((gpu_idx + 1)) +done <<< "$raw" + +# Emit aggregates (only when multiple GPUs; single GPU already emitted above) +mi=0 +for m in "${metrics[@]}"; do + IFS='|' read -r ganglia_name _ type units agg title <<< "$m" + if [[ "$agg" == "none" || $gpu_count -eq 1 ]]; then + mi=$((mi + 1)) + continue + fi + + if [[ "$agg" == "sum" ]]; then + metric "$ganglia_name" "${agg_sum[$mi]}" "$type" "$units" "Total $title" + elif [[ "$agg" == "avg" && ${agg_count[$mi]} -gt 0 ]]; then + avg=$(awk "BEGIN {printf \"%.2f\", ${agg_sum[$mi]} / ${agg_count[$mi]}}") + metric "$ganglia_name" "$avg" "$type" "$units" "Average $title" + fi + mi=$((mi + 1)) +done diff --git a/hw2ganglia b/hw2ganglia new file mode 100644 index 0000000..15dc635 --- /dev/null +++ b/hw2ganglia @@ -0,0 +1,40 @@ +#!/bin/bash + +# Environment statistics (UPS load and temperature) +# Load from older UPSs is in tenths of a percent +load_ups1=`snmpget -v 1 -c 'private' -m PowerNet-MIB ups1 PowerNet-MIB::upsHighPrecOutputLoad.0 | grep -Eo '[0-9]+$'` +power_ups1=$((6 * $load_ups1)) +load_ups2=`snmpget -v 1 -c 'private' -m PowerNet-MIB ups2 PowerNet-MIB::upsHighPrecOutputLoad.0 | grep -Eo '[0-9]+$'` +power_ups2=$((6 * $load_ups2)) +# New UPS lets us read power in watts directly +power_ups221=`snmpget -v 1 -c 'r3@dsnmp!' -m PowerNet-MIB ups221 PowerNet-MIB::upsAdvOutputActivePower.0 | grep -Eo '[0-9]+$'` +power_222=$(($power_ups1 + $power_ups2)) +power_total=$(($power_ups1 + $power_ups2 + $power_ups221)) +gmetric --name "power_222" --value "$power_222" --type "int32" --group "environment" --title "222 cluster power consumption" --units "W" +gmetric --name "power_221" --value "$power_ups221" --type "int32" --group "environment" --title "221 cluster power consumption" --units "W" + +# Temperature from universal IO +temp_222=`snmpget -v 1 -c 'private' -m PowerNet-MIB ups1 PowerNet-MIB::uioSensorStatusTemperatureDegF.1.1 | grep -Eo '[0-9]+$'` +temp_221=`snmpget -v 1 -c 'r3@dsnmp!' -m PowerNet-MIB ups221 PowerNet-MIB::uioSensorStatusTemperatureDegF.1 | grep -Eo '[0-9]+$'` +gmetric --name "ambient_temp_222" --value "$temp_222" --type "int8" --group "environment" --title "222 ambient temperature" --units "F" +gmetric --name "ambient_temp_221" --value "$temp_221" --type "int8" --group "environment" --title "221 ambient temperature" --units "F" + +# Bandwidth from the switch cross-cluster +# Note port 10201 is the first 10G SFP port +octet_out1=`snmpget -v 1 -c private switch-ipmi IF-MIB::ifOutOctets.10201|grep -Eo [0-9]+$` +sleep 1 +octet_out2=`snmpget -v 1 -c private switch-ipmi IF-MIB::ifOutOctets.10201|grep -Eo [0-9]+$` +bytes_out=$(($octet_out2-$octet_out1)) +if [ ! $bytes_out -lt 0 ]; then + # Don't send gmetric if 32-bit counter has rolled over and result is negative + gmetric --name "uplink_bytes_out" --value "$bytes_out" --type "uint32" --group "environment" --title "221 uplink bytes out" --units "Bytes/sec" +fi + +octet_in1=`snmpget -v 1 -c private switch-ipmi IF-MIB::ifInOctets.10201|grep -Eo [0-9]+$` +sleep 1 +octet_in2=`snmpget -v 1 -c private switch-ipmi IF-MIB::ifInOctets.10201|grep -Eo [0-9]+$` +bytes_in=$(($octet_in2-$octet_in1)) +if [ ! $bytes_in -lt 0 ]; then + # Don't send gmetric if 32-bit counter has rolled over and result is negative + gmetric --name "uplink_bytes_in" --value "$bytes_in" --type "uint32" --group "environment" --title "221 uplink bytes in" --units "Bytes/sec" +fi diff --git a/infiniband2ganglia b/infiniband2ganglia new file mode 100644 index 0000000..ad4a2dd --- /dev/null +++ b/infiniband2ganglia @@ -0,0 +1,21 @@ +#!/bin/bash + +# Collect statistics on Infiniband networking +# Note the -r flag resets the counters, so this should be ~1 minute of data +if [[ ( -n "`which ibstat`" ) && ( -n "`ibstat`" ) ]]; then + # This node has Infiniband + ib_stats=`perfquery -rx` + if [ $? -eq 0 ]; then + ib_bytes_out=$(grep -Po '^PortXmitData:\.+\K\d+$' <<< "$ib_stats") + ib_bytes_in=$(grep -Po '^PortRcvData:\.+\K\d+$' <<< "$ib_stats") + ib_pkts_out=$(grep -Po '^PortXmitPkts:\.+\K\d+$' <<< "$ib_stats") + ib_pkts_in=$(grep -Po '^PortRcvPkts:\.+\K\d+$' <<< "$ib_stats") + + # use counter * 4 / 60 because running once a minute, and perfquery returns 'octets divided by 4' for some reason + # we also use a float because gmetric only supports up to uint32, and IB can do more than 4GB/s = 16 Gbit + gmetric --name="ib_bytes_out" --title="InfiniBand Bytes Sent" --value="$(($ib_bytes_out * 4 / 60))" --units="bytes/sec" --type="float" --group="network" --conf=$conf + gmetric --name="ib_bytes_in" --title="InfiniBand Bytes Received" --value="$(($ib_bytes_in * 4 / 60))" --units="bytes/sec" --type="float" --group="network" --conf=$conf + gmetric --name="ib_pkts_out" --title="InfiniBand Packets Sent" --value="$(($ib_pkts_out / 60))" --units="packets/sec" --type="float" --group="network" --conf=$conf + gmetric --name="ib_pkts_in" --title="InfiniBand Packets Received" --value="$(($ib_pkts_out / 60))" --units="packets/sec" --type="float" --group="network" --conf=$conf + fi +fi \ No newline at end of file diff --git a/ipmisensors2ganglia b/ipmisensors2ganglia new file mode 100644 index 0000000..8737ede --- /dev/null +++ b/ipmisensors2ganglia @@ -0,0 +1,56 @@ +#!/bin/bash + +# XML escape in string metrics. Gmond handles this for its own metrics, we must do it ourself +xml_escape() { + local input="$1" + input="${input//&/\&}" + input="${input///\>}" + input="${input//\"/\"}" + input="${input//\'/\'}" + echo "$input" +} + +# Read all the IPMI hardware sensors +/usr/sbin/ipmi-sensors --ignore-not-available --no-header-output | + sed -Es 's/ *\| */|/g' | +while read line; do + # Read the line into a split array + readarray -td '|' linearray < <(echo $line) + + # line looks like: + # 4 | CPU1 Temp | Temperature | 46.00 | C | 'OK' + + # name of the sensor + reading_name=${linearray[1]} + + # replace spaces with underscores + reading_name=${reading_name// /_} + + # fix the tilde on some machines which report ram temperatures in banks + reading_name=${reading_name//\~/_thru_} + + # make it all lowercase + reading_name=${reading_name,,*} + + # reading units and value + reading_value=${linearray[3]} + reading_units=${linearray[4]} + + # If it's an event, report a string not a float + if [ $reading_value == "N/A" ]; then + reading_event=${linearray[5]} + reading_event=${reading_event//$'\n'/} + reading_event=${reading_event//"' '"/, } + reading_event=${reading_event//"'"/} + gmetric --name="$reading_name" --value="$(xml_escape "$reading_event")" --type=string --group=physical --conf=$conf + else + gmetric --name="$reading_name" --value="$reading_value" --units=$reading_units --type=float --group=physical --conf=$conf + fi + + # echo "$reading_name $reading_value $reading_units" +done + + + + diff --git a/psi2ganglia b/psi2ganglia new file mode 100755 index 0000000..5174e54 --- /dev/null +++ b/psi2ganglia @@ -0,0 +1,25 @@ +#!/bin/bash +for res in /proc/pressure/*; do + res_name=$(basename $res) + res_name_nice=${res_name@U} + res_name_nice=${res_name_nice/MEMORY/Memory} + while read line; do + for v in $line; do + if [[ $v == "some" || $v == "full" ]]; then + somefull=$v + elif [[ $v == avg* || $v == total* ]]; then + timespan=${v%%=*} + timespan=${timespan#avg} + val=${v#*=} + if [[ $timespan == "total" ]]; then + timespan_nice="Total" + units="usec" + else + timespan_nice="$timespan Second Average" + units="%" + fi + gmetric --name="psi_${somefull}_${res_name}_${timespan}" --title="PSI ${somefull@u} ${res_name_nice} ${timespan_nice}" --value="$val" --type="float" --group="load" --units="$units" + fi + done + done < "$res" +done \ No newline at end of file diff --git a/sensors2ganglia b/sensors2ganglia new file mode 100755 index 0000000..c5ca1ac --- /dev/null +++ b/sensors2ganglia @@ -0,0 +1,103 @@ +#!/bin/bash + +while read line; do + # Check if this is a new batch of sensors and get the name to set a prefix + if [[ $(grep -Po '^\S+$' <<< "$line") ]]; then + OLDIFS=$IFS + IFS='-' + read -a splitline <<< "$line" + IFS=$OLDIFS + + # Make a prefix for duplicate sensors + case ${splitline[0]} in + nvme) + prefix=${splitline[0]}_${splitline[2]}_ + ;; + + jc42) + prefix=${splitline[0]}_${splitline[3]}_ + ;; + + acpitz) + prefix=${splitline[0]}_ + ;; + + *) + prefix="" + ;; + esac + fi + + # Process the sensor readings + reading_name=$(grep -Po '^.*?(?=:)' <<< "$line") + reading_val=$(grep -Po ':\s*\+?\K-?[0-9.]+' <<< "$line") + reading_units=$(grep -Po ':\s*\+?-?[0-9.]+\s?\K[^0-9. ]+' <<< "$line") + + # Make reading name lowercase and convert spaces to underscore + reading_name=${reading_name,,*} + reading_name=${reading_name// /_} + # Fixes for weird characters + reading_name=${reading_name//+/} + reading_name=${reading_name//-/minus} + # Remove some extra unit markings from the name + # Zenpower + reading_name=${reading_name//_p_/_} + reading_name=${reading_name//_c_/_} + # Others + reading_name=${reading_name//_temp/} + reading_name=${reading_name//temp1/1} + reading_name=${reading_name//temp2/2} + reading_name=${reading_name//temp3/3} # Just in case + + # Ganglia does not like the degrees symbol + reading_units=${reading_units//°/} + + # Suffix a description of the type of reading + case $reading_units in + V | mV) + suffix="_voltage" + ;; + + RPM) + suffix="_speed" + ;; + + C) + suffix="_temp" + ;; + + A) + suffix="_current" + ;; + + W) + suffix="_power" + ;; + + GHz) + suffix="_freq" + ;; + + *) + suffix="" + ;; + esac + + # Ignore some unset/unconnected values + if [[ "$reading_val" ]]; then + case $reading_val in + -40.0 | 0.0 | 0.00 | 0 | 127.0) + ;; + + *) + # Log the reading + if [[ "$1" == "debug" ]]; then + echo $prefix$reading_name$suffix = $reading_val : $reading_units + else + gmetric --name="$prefix$reading_name$suffix" --value="$reading_val" --units="$reading_units" --type=float --group=sensors + fi + + ;; + esac + fi +done <<<$(sensors;/usr/local/bin/zen-rapl) diff --git a/slurmnode2ganglia b/slurmnode2ganglia new file mode 100644 index 0000000..4ea8ef8 --- /dev/null +++ b/slurmnode2ganglia @@ -0,0 +1,60 @@ +#!/bin/bash + +# XML escape in string metrics. Gmond handles this for its own metrics, we must do it ourself +xml_escape() { + local input="$1" + input="${input//&/\&}" + input="${input///\>}" + input="${input//\"/\"}" + input="${input//\'/\'}" + echo "$input" +} + +hn=$(hostname) + +# Only run Slurm checks on nodes with slurm installed +if [ $(which sinfo) ]; then + # Slurm node state + slurm_node_state=$(sinfo --nodes=$hn --Node --noheader --format='%t' | head -n1) + if [ "$slurm_node_state" ]; then + gmetric --name="slurm_node_state" --title="Slurm node state" --value="$(xml_escape "$slurm_node_state")" --type="string" --group="slurm" --conf=$conf + fi + + # Slurm reason for being down + slurm_down_reason=$(sinfo --nodes=$hn --Node --noheader --format='%E' | head -n1) + if [ "$slurm_down_reason" ]; then + gmetric --name="slurm_down_reason" --title="Slurm node down reason" --value="$(xml_escape "$slurm_down_reason")" --type="string" --group="slurm" --conf=$conf + fi + + # Slurm partition + while read line; do + slurm_partition=$slurm_partition$line, + done <<< $(sinfo --nodes=$hn --Node --noheader --format='%R') + slurm_partition=${slurm_partition%,} + if [ $slurm_partition ]; then + gmetric --name="slurm_partition" --title="Slurm partition" --value="$(xml_escape "$slurm_partition")" --type="string" --group="slurm" --conf=$conf + fi + + # Slurm jobs on this node + node_job_output=$(squeue --nodes=$hn --format='%i|%j|%u|%M|' --noheader --sort='i' 2>/dev/null) + if [ $? -eq 0 ]; then + while read line; do + readarray -td '|' linearray <<< ${line} + slurm_job_id=${slurm_job_id}${linearray[0]}, + slurm_job_name=${slurm_job_name}${linearray[1]}, + slurm_job_user=${slurm_job_user}${linearray[2]}, + slurm_job_time_elapsed=${slurm_job_time_elapsed}${linearray[3]%}, + done <<< ${node_job_output} + slurm_job_id=${slurm_job_id%,} + slurm_job_name=${slurm_job_name%,} + slurm_job_user=${slurm_job_user%,} + slurm_job_time_elapsed=${slurm_job_time_elapsed%,} + + gmetric --name="slurm_job_id" --title="Slurm running job ID" --value="$(xml_escape "$slurm_job_id")" --type="string" --group="slurm" --conf=$conf + gmetric --name="slurm_job_name" --title="Slurm running job name" --value="$(xml_escape "$slurm_job_name")" --type="string" --group="slurm" --conf=$conf + gmetric --name="slurm_job_user" --title="Slurm running job user" --value="$(xml_escape "$slurm_job_user")" --type="string" --group="slurm" --conf=$conf + gmetric --name="slurm_job_time_elapsed" --title="Slurm running job elapsed time" --value="$(xml_escape "$slurm_job_time_elapsed")" --type="string" --group="slurm" --conf=$conf + fi + +fi \ No newline at end of file diff --git a/slurmqueue2ganglia b/slurmqueue2ganglia new file mode 100644 index 0000000..7e46b87 --- /dev/null +++ b/slurmqueue2ganglia @@ -0,0 +1,44 @@ +#!/bin/bash + +# Gather statistics on the queue +queue_state=`squeue --all --noheader --Format State` +jobs_running=`grep RUNNING <<< $queue_state | wc --lines` +jobs_pending=`grep PENDING <<< $queue_state | wc --lines` +if [ -z "$queue_state" ]; then + jobs_total="0" +else + jobs_total=`wc --lines <<< $queue_state` +fi +jobs_other=$(($jobs_total - $jobs_pending - $jobs_running)) + +gmetric --name "slurm_jobs_running" --value $jobs_running --type uint16 --group slurm --title "Jobs running" --units "Jobs" +gmetric --name "slurm_jobs_pending" --value $jobs_pending --type uint16 --group slurm --title "Jobs pending" --units "Jobs" +gmetric --name "slurm_jobs_other" --value $jobs_other --type uint16 --group slurm --title "Jobs other" --units "Jobs" + +# Statistics on nodes in use +nodes_alloc_total=0 +for part in `sinfo --format="%R" --noheader`; do + nodes_alloc=`sinfo --partition $part --Format NodeAI|grep -Eo '^[0-9]+'` + gmetric --name "slurm_alloc_$part" --value "$nodes_alloc" --type uint16 --group slurm --title "$part nodes allocated" --units "Nodes" + if [ ! -z $nodes_alloc ]; then + nodes_alloc_total=$(($nodes_alloc_total + $nodes_alloc)) + fi +done +gmetric --name "slurm_alloc_total" --value "$nodes_alloc_total" --type uint16 --group slurm --title "Total nodes allocated" --units "Nodes" + +# Job with longest time in queue (that is still pending) +# Long statement here because we don't want to count jobs that are pending because of a dependency (not waiting on resources) +pending_time_seconds=`squeue --noheader --all --Format 'Reason,PendingTime' --states PENDING | + grep --invert-match '^Dependency' | + grep --invert-match '^Licenses' | + grep --invert-match '^JobHeld' | + grep -Po '\d+' | + sort --numeric --reverse | + head -n 1` +if [ -z $pending_time_seconds ]; then + # No pending jobs + pending_time_hours=0 +else + pending_time_hours=`bc -l <<< "$pending_time_seconds / 3600"`; +fi +gmetric --name "slurm_job_oldest" --value "$pending_time_hours" --type float --group slurm --title "Oldest pending job" --units "Hours" diff --git a/smart2ganglia b/smart2ganglia new file mode 100755 index 0000000..9be3af5 --- /dev/null +++ b/smart2ganglia @@ -0,0 +1,15 @@ +#!/bin/bash + +# Loop through all hard drives and read their temperatures +for DEVICE in /dev/sd[a-z]; do + # Check if the device is a disk (not a partition) + if [ -b "$DEVICE" ]; then + # Get the temperature from smartctl + TEMPERATURE=$(/usr/sbin/smartctl -A $DEVICE | grep "Temperature_Celsius" | awk '{print $10}') + + # If the temperature is not empty, push it to Ganglia using gmetric + if [ -n "$TEMPERATURE" ]; then + gmetric --name ${DEVICE##*/}_temp --value $TEMPERATURE --type float --units C --group=sensors + fi + fi +done diff --git a/systemd2ganglia b/systemd2ganglia new file mode 100644 index 0000000..d4399de --- /dev/null +++ b/systemd2ganglia @@ -0,0 +1,5 @@ +#!/bin/bash + +# systemd statistics +systemd_units_failed=$(systemctl --state=failed|grep -Po '\d+(?= loaded units listed.)') +gmetric --name="systemd_units_failed" --title="systemd failed units" --value="$systemd_units_failed" --type="uint16" --group="process" --conf=$conf diff --git a/ups2ganglia b/ups2ganglia new file mode 100755 index 0000000..a0a5c0b --- /dev/null +++ b/ups2ganglia @@ -0,0 +1,45 @@ +#!/bin/bash + +IFS=':' + +while read -a line; do + # Clear out extra spaces + key=$(tr -d ' ' <<< ${line[0]}) + val=${line[1]## } + val_num=$(grep -Eo '[0-9.]+' <<< $val) + + case $key in + MODEL) + gmetric --name="ups_model" --value="$val" --type=string --group=ups --title="UPS model" + ;; + LINEV) + gmetric --name="ups_line_voltage" --value="$val_num" --units="V" --type=float --group=ups --title="UPS line voltage" + ;; + LOADPCT) + gmetric --name="ups_load_percent" --value="$val_num" --units="%" --type=float --group=ups --title="UPS load" + loadpct=$(tr -d '.' <<< $val_num) # Strip out decimals for bash's integer math (should now be in thousandths, for accuracy in case there is a decimal) + ;; + BCHARGE) + gmetric --name="ups_battery_charge" --value="$val_num" --units="%" --type=float --group=ups --title="UPS battery charge" + ;; + TIMELEFT) + gmetric --name="ups_time_left" --value="$val_num" --units="min" --type=float --group=ups --title="UPS time left" + ;; + BATTV) + gmetric --name="ups_battery_voltage" --value="$val_num" --units="V" --type=float --group=ups --title="UPS battery voltage" + ;; + TONBATT) + gmetric --name="ups_time_on_battery" --value="$val_num" --units="s" --type=uint32 --group=ups --title="UPS time on battery" + ;; + BATTDATE) + gmetric --name="ups_battery_date" --value="$val" --type=string --group=ups --title="UPS battery date" + ;; + NOMPOWER) + nompower=$val_num + ;; + esac +done <<< $(/usr/sbin/apcaccess) + +# Calculate load in watts +load_watts=$(( ($loadpct * $nompower) / 1000 )) +gmetric --name="ups_load" --value="$load_watts" --units="W" --type=float --group=ups --title="UPS load"