Initial commit

slurmqueue2ganglia

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Gather statistics on the queue
+queue_state=`squeue --all --noheader --Format State`
+jobs_running=`grep RUNNING <<< $queue_state | wc --lines`
+jobs_pending=`grep PENDING <<< $queue_state | wc --lines`
+if [ -z "$queue_state" ]; then
+	jobs_total="0"
+else
+	jobs_total=`wc --lines <<< $queue_state`
+fi
+jobs_other=$(($jobs_total - $jobs_pending - $jobs_running))
+
+gmetric --name "slurm_jobs_running" --value $jobs_running --type uint16 --group slurm --title "Jobs running" --units "Jobs"
+gmetric --name "slurm_jobs_pending" --value $jobs_pending --type uint16 --group slurm --title "Jobs pending" --units "Jobs"
+gmetric --name "slurm_jobs_other" --value $jobs_other --type uint16 --group slurm --title "Jobs other" --units "Jobs"
+
+# Statistics on nodes in use
+nodes_alloc_total=0
+for part in `sinfo --format="%R" --noheader`; do
+	nodes_alloc=`sinfo --partition $part --Format NodeAI|grep -Eo '^[0-9]+'`
+	gmetric --name "slurm_alloc_$part" --value "$nodes_alloc" --type uint16 --group slurm --title "$part nodes allocated" --units "Nodes"
+	if [ ! -z $nodes_alloc ]; then
+		nodes_alloc_total=$(($nodes_alloc_total + $nodes_alloc))
+	fi
+done
+gmetric --name "slurm_alloc_total" --value "$nodes_alloc_total" --type uint16 --group slurm --title "Total nodes allocated" --units "Nodes"
+
+# Job with longest time in queue (that is still pending)
+# Long statement here because we don't want to count jobs that are pending because of a dependency (not waiting on resources)
+pending_time_seconds=`squeue --noheader --all --Format 'Reason,PendingTime' --states PENDING | 
+					  grep --invert-match '^Dependency' | 
+					  grep --invert-match '^Licenses' | 
+					  grep --invert-match '^JobHeld' |
+					  grep -Po '\d+' | 
+					  sort --numeric --reverse | 
+					  head -n 1`
+if [ -z $pending_time_seconds ]; then
+	# No pending jobs
+	pending_time_hours=0
+else	
+	pending_time_hours=`bc -l <<< "$pending_time_seconds / 3600"`;
+fi
+gmetric --name "slurm_job_oldest" --value "$pending_time_hours" --type float --group slurm --title "Oldest pending job" --units "Hours"