11 lines
230 B
Bash
11 lines
230 B
Bash
# Target AWQ-quantized model ID
|
|
MODEL_ID=Qwen/Qwen2.5-3B-Instruct-AWQ
|
|
|
|
# Hugging Face Token (optional, only for gated models)
|
|
HF_TOKEN=""
|
|
|
|
# vLLM inference parameters
|
|
MAX_MODEL_LEN=4096
|
|
GPU_MEMORY_UTILIZATION=0.90
|
|
MAX_NUM_SEQS=64
|