feat: initial commit with vLLM production stack and benchmark tools

2026-09-09 18:22:40 +02:00
commit 20ae3cfb48
10 changed files with 331 additions and 0 deletions
@@ -0,0 +1,10 @@
+# Target AWQ-quantized model ID
+MODEL_ID=Qwen/Qwen2.5-3B-Instruct-AWQ
+
+# Hugging Face Token (optional, only for gated models)
+HF_TOKEN=""
+
+# vLLM inference parameters
+MAX_MODEL_LEN=4096
+GPU_MEMORY_UTILIZATION=0.90
+MAX_NUM_SEQS=64