From ba7d82f982f3b1ed746d9e3e95d86ec3d6ce4aab Mon Sep 17 00:00:00 2001
From: moosecrap <admin@long-cat.net>
Date: Wed, 22 Apr 2026 22:10:04 -0700
Subject: [PATCH] Initial commit

---
 bench-llms.sh      | 48 ++++++++++++++++++++++++++++++++++++++++++++++
 models.txt         |  8 ++++++++
 prefills/ao3.txt   |  3 +++
 prompts/monkey.txt |  1 +
 readme.txt         |  7 +++++++
 5 files changed, 67 insertions(+)
 create mode 100755 bench-llms.sh
 create mode 100644 models.txt
 create mode 100644 prefills/ao3.txt
 create mode 100644 prompts/monkey.txt
 create mode 100644 readme.txt

diff --git a/bench-llms.sh b/bench-llms.sh
new file mode 100755
index 0000000..f1b89a4
--- /dev/null
+++ b/bench-llms.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+NORMAL_REQ='{model: $model, messages:[{role: "user", content: $prompt}]}'
+PREFILL_REQ='{model: $model, chat_template_kwargs: {"enable_thinking": false}, messages:[{role: "user", content: $prompt}, {role: "assistant", content: $prefill}]}'
+
+ENDPOINT='http://localhost:8080/v1/chat/completions'
+WORDLIST='/etc/dictionaries-common/words'
+
+get_response() {
+	model=$1
+	prompt_file=$2
+	prefill_file=$3
+	if [ -z $prefill_file ]; then
+		req=$(jq --null-input --arg model $model --rawfile prompt $prompt_file "$NORMAL_REQ")
+	else
+		cat $prefill_file
+		req=$(jq --null-input --arg model $model --rawfile prompt $prompt_file --rawfile prefill $prefill_file "$PREFILL_REQ")
+	fi
+
+	resp=$(curl -X POST --url $ENDPOINT --silent --data @- <<< $req)
+	jq --raw-output --exit-status .choices[0].message.content <<< $resp 2>&1
+	
+	if [ $? -ne 0 ]; then
+		echo Error: $(jq --raw-output .error.message <<< $resp)
+	fi
+}
+
+sort --random-sort models.txt | while read model; do
+	random_name=$(grep -Po '^[A-Z][a-z]+$' $WORDLIST | sort --random-sort | head -n1)
+	random_name=${random_name,}
+	echo $random_name : $model >> key.txt
+	for prompt_file in prompts/*.txt; do
+		promptname=$(basename ${prompt_file} .txt)
+		mkdir -p responses/$promptname
+		for prefill_file in prefills/*.txt; do
+			if [ ! -f $prefill_file ]; then
+				# no prefill
+				prefill_file=""
+				prefill="."
+			else
+				# prefill included
+				prefill=$(basename $prefill_file .txt)
+			fi
+			echo $(date +%T) $random_name: $promptname $prefill
+			mkdir -p responses/$promptname/$prefill
+			get_response $model $prompt_file $prefill_file > responses/$promptname/$prefill/$random_name.txt
+		done
+	done
+done
\ No newline at end of file
diff --git a/models.txt b/models.txt
new file mode 100644
index 0000000..191471b
--- /dev/null
+++ b/models.txt
@@ -0,0 +1,8 @@
+NVIDIA-Nemotron-3-Super-120B-A12B-MXFP4_MOE
+gemma-4-31B-it-BF16
+GLM-4.5-Air-UD-Q5_K_XL
+c4ai-command-r-plus-08-2024-Q6_K
+Devstral-2-123B-Instruct-2512-UD-Q5_K_XL
+mistral-nemo_12b-instruct-2407-fp16
+Ministral-3-14B-Instruct-2512-BF16
+Mistral-Large-Instruct-2411-Q5_K_M
diff --git a/prefills/ao3.txt b/prefills/ao3.txt
new file mode 100644
index 0000000..abe5332
--- /dev/null
+++ b/prefills/ao3.txt
@@ -0,0 +1,3 @@
+Rating: Explicit
+Archive warnings: No Archive Warnings Apply
+Categories: 
\ No newline at end of file
diff --git a/prompts/monkey.txt b/prompts/monkey.txt
new file mode 100644
index 0000000..8d7b9e0
--- /dev/null
+++ b/prompts/monkey.txt
@@ -0,0 +1 @@
+Tell me 10 jokes about monkeys!
diff --git a/readme.txt b/readme.txt
new file mode 100644
index 0000000..a06a03a
--- /dev/null
+++ b/readme.txt
@@ -0,0 +1,7 @@
+Requirements:
+* llama.cpp or another OpenAI-compatible server
+* bash
+* curl
+* jq
+
+`./llama-server --models-preset ~/models/preset.ini --models-dir ~/models/ --models-max 1`
\ No newline at end of file