vllm
serve
/opt/local-models/qwen36-27b-f16-gguf/Qwen3.6-27B-F16.gguf
--served-model-name
Qwen3.6-27B-F16-GGUF-lmheadunquant
--enable-auto-tool-choice
--tool-call-parser
qwen3_coder
--dtype
half
--host
0.0.0.0
--port
8001
--tensor-parallel-size
8
--max-model-len
131072
--gpu-memory-utilization
0.95
--trust-remote-code
--generation-config
vllm
-O=3
--reasoning-parser
qwen3
--async-scheduling
--load-format
gguf
--tokenizer
Qwen/Qwen3.6-27B
--tokenizer-revision
6a9e13bd6fc8f0983b9b99948120bc37f49c13e9
--hf-config-path
/opt/local-models/qwen36-27b-text-config-eosfix
--hf-overrides
{"architectures":["Qwen3_5ForCausalLM"]}
--language-model-only
--max-num-seqs
2
--max-num-batched-tokens
4
--no-enable-prefix-caching
--compilation-config
{"debug_dump_path":"/runwork/debug_dump","cache_dir":"/runwork/compile_cache","compile_cache_save_format":"unpacked","use_inductor_graph_partition":true}
