#!/bin/bash

# Gather statistics on the queue
queue_state=`squeue --all --noheader --Format State`
jobs_running=`grep RUNNING <<< $queue_state | wc --lines`
jobs_pending=`grep PENDING <<< $queue_state | wc --lines`
if [ -z "$queue_state" ]; then
	jobs_total="0"
else
	jobs_total=`wc --lines <<< $queue_state`
fi
jobs_other=$(($jobs_total - $jobs_pending - $jobs_running))

gmetric --name "slurm_jobs_running" --value $jobs_running --type uint16 --group slurm --title "Jobs running" --units "Jobs"
gmetric --name "slurm_jobs_pending" --value $jobs_pending --type uint16 --group slurm --title "Jobs pending" --units "Jobs"
gmetric --name "slurm_jobs_other" --value $jobs_other --type uint16 --group slurm --title "Jobs other" --units "Jobs"

# Statistics on nodes in use
nodes_alloc_total=0
for part in `sinfo --format="%R" --noheader`; do
	nodes_alloc=`sinfo --partition $part --Format NodeAI|grep -Eo '^[0-9]+'`
	gmetric --name "slurm_alloc_$part" --value "$nodes_alloc" --type uint16 --group slurm --title "$part nodes allocated" --units "Nodes"
	if [ ! -z $nodes_alloc ]; then
		nodes_alloc_total=$(($nodes_alloc_total + $nodes_alloc))
	fi
done
gmetric --name "slurm_alloc_total" --value "$nodes_alloc_total" --type uint16 --group slurm --title "Total nodes allocated" --units "Nodes"

# Job with longest time in queue (that is still pending)
# Long statement here because we don't want to count jobs that are pending because of a dependency (not waiting on resources)
pending_time_seconds=`squeue --noheader --all --Format 'Reason,PendingTime' --states PENDING | 
					  grep --invert-match '^Dependency' | 
					  grep --invert-match '^Licenses' | 
					  grep --invert-match '^JobHeld' |
					  grep -Po '\d+' | 
					  sort --numeric --reverse | 
					  head -n 1`
if [ -z $pending_time_seconds ]; then
	# No pending jobs
	pending_time_hours=0
else	
	pending_time_hours=`bc -l <<< "$pending_time_seconds / 3600"`;
fi
gmetric --name "slurm_job_oldest" --value "$pending_time_hours" --type float --group slurm --title "Oldest pending job" --units "Hours"