{ "hf_id": "Qwen/Qwen3.6-27B", "meta": { "title": "Qwen3.6-27B", "slug": "Qwen3.6-27b", "provider": "Qwen", "description": "Qwen3.6 dense multimodal model (27B) with gated delta networks hybrid attention, MTP, and 262K context", "date_added": "2026-04-22", "date_updated": "2026-10-12", "tasks": [ "multimodal", "text" ], "performance_headline": "Qwen3.6 flagship dense — single-GPU FP8, 2x GPU BF16, 2-4x Intel Arc Pro B60/B70 BF16, Ascend 910C TP2+DP8, or Ascend 950DT TP1+DP8", "related_recipes": [ "Qwen/Qwen3.6-35B-A3B" ], "hardware": { "dgx_spark_gb10": "verified", "rtx_pro_6000": "verified", "dgx_station_gb300": "verified", "rtx_5090": "verified", "ascend_950dt": "verified", "ascend_910c": "verified", "arc_pro_b60": "verified", "arc_pro_b70": "verified", "mi300x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_USE_RUST_FRONTEND": "1" }, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve Qwen/Qwen3.6-27B \\\n --trust-remote-code \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser qwen3 \\\n --mm-encoder-tp-mode data", "argv": [ "vllm", "serve", "Qwen/Qwen3.6-27B", "--trust-remote-code", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "qwen3", "--mm-encoder-tp-mode", "data" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n vllm/vllm-openai:latest Qwen/Qwen3.6-27B \\\n --trust-remote-code \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser qwen3 \\\n --mm-encoder-tp-mode data", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_USE_RUST_FRONTEND=1", "vllm/vllm-openai:latest", "Qwen/Qwen3.6-27B", "--trust-remote-code", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "qwen3", "--mm-encoder-tp-mode", "data" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "multi_node_tp": "/Qwen/Qwen3.6-27B/strategies/multi_node_tp.json", "kv_store_centralized_mooncake": "/Qwen/Qwen3.6-27B/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/Qwen/Qwen3.6-27B/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/Qwen/Qwen3.6-27B/hw/h200.json", "h100": "/Qwen/Qwen3.6-27B/hw/h100.json", "b200": "/Qwen/Qwen3.6-27B/hw/b200.json", "gb200": "/Qwen/Qwen3.6-27B/hw/gb200.json", "b300": "/Qwen/Qwen3.6-27B/hw/b300.json", "gb300": "/Qwen/Qwen3.6-27B/hw/gb300.json", "dgx_station_gb300": "/Qwen/Qwen3.6-27B/hw/dgx_station_gb300.json", "dgx_spark_gb10": "/Qwen/Qwen3.6-27B/hw/dgx_spark_gb10.json", "rtx_pro_6000": "/Qwen/Qwen3.6-27B/hw/rtx_pro_6000.json", "rtx_5090": "/Qwen/Qwen3.6-27B/hw/rtx_5090.json", "mi300x": "/Qwen/Qwen3.6-27B/hw/mi300x.json", "mi325x": "/Qwen/Qwen3.6-27B/hw/mi325x.json", "mi350x": "/Qwen/Qwen3.6-27B/hw/mi350x.json", "mi355x": "/Qwen/Qwen3.6-27B/hw/mi355x.json", "arc_pro_b60": "/Qwen/Qwen3.6-27B/hw/arc_pro_b60.json", "arc_pro_b70": "/Qwen/Qwen3.6-27B/hw/arc_pro_b70.json", "ascend_950dt": "/Qwen/Qwen3.6-27B/hw/ascend_950dt.json", "ascend_910c": "/Qwen/Qwen3.6-27B/hw/ascend_910c.json" } }, "model": { "model_id": "Qwen/Qwen3.6-27B", "min_vllm_version": "0.17.0", "default_frontend": "rust", "architecture": "dense", "parameter_count": "27B", "active_parameters": "27B", "context_length": 262144, "docker_image": { "amd": "vllm/vllm-openai-rocm:v0.31.0" }, "base_args": [ "--trust-remote-code" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable automatic tool choice with Qwen3 Coder parser", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder" ] }, "reasoning": { "description": "Enable chain-of-thought reasoning with Qwen3 parser", "args": [ "--reasoning-parser", "qwen3" ] }, "spec_decoding": { "description": "Speculative decoding — pick native MTP or the Ascend MTP mode.", "default_mode": "mtp", "modes": { "mtp": { "label": "MTP", "description": "Native MTP head, 3 draft tokens.", "variants": [ "default", "fp8", "nvfp4", "redhat_fp8" ], "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" ], "hardware_overrides": { "dgx_spark_gb10": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":5}" ] } } }, "qwen3_5_mtp": { "label": "MTP (Ascend)", "description": "MTP mode documented for vLLM Ascend.", "variants": [ "ascend_w8a8_mxfp8", "ascend_w8a8" ], "args": [ "--speculative-config", "{\"method\":\"qwen3_5_mtp\",\"num_speculative_tokens\":3,\"enforce_eager\":true}" ] } } }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "encoder_parallel": { "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.", "args": [ "--mm-encoder-tp-mode", "data" ] }, "long_context": { "description": "YaRN RoPE scaling past the native 262,144-token window to ~1,010,000 tokens. Static YaRN holds the scaling factor constant regardless of input length, so leave this off unless you actually serve long prompts; for ~524K, halve the factor to 2.0.", "args": [ "--hf-overrides", "{\"text_config\": {\"rope_parameters\": {\"mrope_interleaved\": true, \"mrope_section\": [11, 11, 10], \"rope_type\": \"yarn\", \"rope_theta\": 10000000, \"partial_rotary_factor\": 0.25, \"factor\": 4.0, \"original_max_position_embeddings\": 262144}}}", "--max-model-len", "1010000" ], "env": { "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1" } } }, "opt_in_features": [ "spec_decoding", "text_only", "long_context" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 65, "description": "Full precision BF16 — fits on 1x H200, 2x H100, 4x Intel Arc Pro B60, 2x Intel Arc Pro B70, or an Ascend A3 node (TP2+DP8)", "tp": { "ascend_910c": 2 }, "hardware_overrides": { "rtx_pro_6000": { "extra_args": [ "--max-num-seqs", "8" ] }, "ascend_910c": { "docker_image": "swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:v0.23.0-a3", "extra_args": [ "--data-parallel-size", "8", "--served-model-name", "qwen3.6", "--max-num-seqs", "32", "--max-model-len", "133000", "--max-num-batched-tokens", "16384", "--gpu-memory-utilization", "0.90", "--no-enable-prefix-caching", "--compilation-config", "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}", "--async-scheduling" ], "extra_env": { "VLLM_USE_MODELSCOPE": "True", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "HCCL_BUFFSIZE": "512", "OMP_PROC_BIND": "false", "OMP_NUM_THREADS": "1", "TASK_QUEUE_ENABLE": "1" } } } }, "fp8": { "model_id": "Qwen/Qwen3.6-27B-FP8", "precision": "fp8", "vram_minimum_gb": 33, "description": "Qwen official FP8 checkpoint — single 40 GB GPU", "extra_args": [ "--max-num-seqs", "512" ], "hardware_overrides": { "dgx_spark_gb10": { "extra_env": { "VLLM_USE_DEEP_GEMM": "0" } }, "rtx_pro_6000": { "extra_env": { "VLLM_USE_DEEP_GEMM": "0" } } }, "json": "/Qwen/Qwen3.6-27B-FP8.json" }, "nvfp4": { "model_id": "nvidia/Qwen3.6-27B-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 27, "tp": 1, "min_vllm_version": "0.28.0", "nightly_required": true, "description": "NVIDIA ModelOpt NVFP4 checkpoint — MLP linears quantized to NVFP4, attention linears and KV cache in FP8. Single Blackwell GPU, including DGX Spark.", "hardware_overrides": { "dgx_station_gb300": { "extra_args": [ "--kv-cache-dtype", "fp8", "--attention-backend", "flashinfer", "--gpu-memory-utilization", "0.92", "--max-model-len", "262144", "--max-num-batched-tokens", "8192", "--enable-chunked-prefill", "--enable-prefix-caching", "--load-format", "fastsafetensors", "--async-scheduling" ] }, "dgx_spark_gb10": { "extra_args": [ "--kv-cache-dtype", "fp8", "--attention-backend", "flashinfer", "--gpu-memory-utilization", "0.5", "--max-model-len", "262144", "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--enable-chunked-prefill", "--enable-prefix-caching", "--load-format", "fastsafetensors", "--async-scheduling" ] }, "rtx_pro_6000": { "extra_args": [ "--kv-cache-dtype", "fp8", "--attention-backend", "flashinfer", "--gpu-memory-utilization", "0.9", "--max-model-len", "262144", "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--enable-chunked-prefill", "--enable-prefix-caching", "--load-format", "fastsafetensors", "--async-scheduling" ] }, "rtx_5090": { "extra_args": [ "--kv-cache-dtype", "fp8", "--attention-backend", "FLASHINFER", "--attention-config", "{\"use_trtllm_attention\": true}", "--gpu-memory-utilization", "0.93", "--max-model-len", "36608", "--max-num-seqs", "8", "--max-num-batched-tokens", "4096", "--gpu-memory-utilization", "0.95", "--enable-chunked-prefill", "--enable-prefix-caching", "--async-scheduling", "--skip-mm-profiling" ], "extra_env": { "VLLM_HAS_FLASHINFER_CUBIN": "1" } } }, "json": "/nvidia/Qwen3.6-27B-NVFP4.json" }, "redhat_fp8": { "model_id": "RedHatAI/Qwen3.6-27B-FP8", "precision": "fp8", "label": "FP8 (RedHat)", "vram_minimum_gb": 33, "description": "RedHatAI FP8 dynamic quantization", "json": "/RedHatAI/Qwen3.6-27B-FP8.json" }, "ascend_w8a8_mxfp8": { "model_id": "Eco-Tech/Qwen3.6-27B-w8a8-mxfp8", "precision": "mxfp8", "label": "W8A8 (Ascend)", "vram_minimum_gb": 34, "supported_hardware": [ "ascend_950dt" ], "tp": 1, "min_vllm_version": "0.23.0", "default_modes": { "spec_decoding": "qwen3_5_mtp" }, "extra_args": [ "--distributed-executor-backend", "mp", "--data-parallel-size", "8", "--quantization", "ascend", "--served-model-name", "qwen3.6", "--max-num-seqs", "32", "--max-model-len", "262144", "--max-num-batched-tokens", "16384", "--gpu-memory-utilization", "0.90", "--no-enable-prefix-caching", "--compilation-config", "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}", "--additional-config", "{\"enable_cpu_binding\":true}" ], "extra_env": { "VLLM_USE_MODELSCOPE": "True", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "HCCL_BUFFSIZE": "512", "OMP_PROC_BIND": "false", "OMP_NUM_THREADS": "1", "TASK_QUEUE_ENABLE": "1" }, "hardware_overrides": { "ascend_950dt": { "docker_image": "quay.io/ascend/vllm-ascend:v0.23.0-a5" } }, "json": "/Eco-Tech/Qwen3.6-27B-w8a8-mxfp8.json" }, "ascend_w8a8": { "model_id": "Eco-Tech/Qwen3.6-27B-w8a8", "precision": "int8", "label": "W8A8 (Ascend A3)", "vram_minimum_gb": 33, "supported_hardware": [ "ascend_910c" ], "tp": 2, "min_vllm_version": "0.23.0", "default_modes": { "spec_decoding": "qwen3_5_mtp" }, "extra_args": [ "--data-parallel-size", "8", "--quantization", "ascend", "--served-model-name", "qwen3.6", "--max-num-seqs", "32", "--max-model-len", "133000", "--max-num-batched-tokens", "16384", "--gpu-memory-utilization", "0.90", "--no-enable-prefix-caching", "--compilation-config", "{\"cudagraph_mode\":\"FULL_DECODE_ONLY\"}", "--additional-config", "{\"enable_cpu_binding\":true}", "--async-scheduling" ], "extra_env": { "VLLM_USE_MODELSCOPE": "True", "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True", "HCCL_BUFFSIZE": "512", "OMP_PROC_BIND": "false", "OMP_NUM_THREADS": "1", "TASK_QUEUE_ENABLE": "1" }, "hardware_overrides": { "ascend_910c": { "docker_image": "swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend:v0.23.0-a3" } }, "description": "ModelSlim INT8 W8A8 checkpoint for one Atlas 800 A3 node (16x Ascend 910C), using TP2+DP8. Requires --quantization ascend.", "json": "/Eco-Tech/Qwen3.6-27B-w8a8.json" } }, "hardware_overrides": { "amd": { "extra_args": [ "--attention-backend", "ROCM_AITER_FA" ], "extra_env": { "VLLM_ROCM_USE_AITER": "1" } }, "xpu": { "extra_args": [ "--enforce-eager" ] } }, "guide": "## Overview\n\n[Qwen3.6-27B](https://huggingface.co/Qwen/Qwen3.6-27B) is the flagship dense\nmodel of the Qwen3.6 family. It uses the same gated delta networks hybrid\nattention as its MoE siblings, supports vision+text input, and natively serves\n262K context. MTP (multi-token prediction) is supported out of the box for\nlow-latency decoding.\n\n## Prerequisites\n\n- **vLLM version:** >= 0.17.0\n- **NVFP4 vLLM version:** >= 0.28.0 (currently nightly). 0.24.0 loads the\n checkpoint, but FlashInfer only selects its XQA decode kernel on SM120\n GPUs from 0.28.0.\n- **Hardware (BF16):** 1x H200, 2x H100, 4x Intel Arc Pro B60, 2x Intel Arc Pro B70\n- **Hardware (FP8):** single 40 GB GPU (H100/H200/L40S)\n- **Hardware (Int4):** single 24 GB GPU\n- **Hardware (NVFP4):** single NVIDIA Blackwell GPU (B200/B300/GB10 DGX Spark)\n- **Hardware (AMD):** 1x MI300X (BF16 fits one 192 GB GPU)\n\n### Install vLLM\n\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend=auto\n```\n\n### Docker\n```bash\ndocker pull vllm/vllm-openai-xpu:latest # Intel XPU (B60 / B70)\n```\n\n## Launching the Server\n\n### Single-GPU FP8\n\n```bash\nvllm serve Qwen/Qwen3.6-27B-FP8 \\\n --max-model-len 262144 \\\n --reasoning-parser qwen3\n```\n\n### BF16 on 2xH100 (TP2)\n\n```bash\nvllm serve Qwen/Qwen3.6-27B \\\n --tensor-parallel-size 2 \\\n --max-model-len 262144 \\\n --reasoning-parser qwen3\n```\n\n### MTP speculative decoding\n\n```bash\nvllm serve Qwen/Qwen3.6-27B-FP8 \\\n --speculative-config '{\"method\": \"mtp\", \"num_speculative_tokens\": 1}' \\\n --reasoning-parser qwen3\n```\n\n### Text-only (skip vision encoder)\n\n```bash\nvllm serve Qwen/Qwen3.6-27B-FP8 \\\n --language-model-only \\\n --reasoning-parser qwen3 \\\n --enable-prefix-caching\n```\n\n### NVFP4 on Blackwell\n\nThe [`nvidia/Qwen3.6-27B-NVFP4`](https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4)\ncheckpoint is NVIDIA's ModelOpt re-quantization: MLP linears drop to NVFP4\n(W4A16) while the attention linears and KV cache stay FP8, so the ~22 GB\nweights fit a single Blackwell GPU. vLLM auto-detects the ModelOpt\nquantization from the checkpoint, so no `--quantization` flag is needed, but\nit does need vLLM 0.28.0+ (currently nightly) for FlashInfer's XQA decode path\non SM120. NVIDIA reports near-lossless accuracy versus the FP8 baseline.\n\nSelect the NVFP4 variant in the command builder above and switch the hardware\npill for the validated single-node launch on DGX Spark (GB10), DGX Station\n(GB300), RTX PRO 6000 or RTX 5090. Memory utilization, context length, batch\nsize, attention backend and MTP depth all differ per box and are encoded per\nhardware, so the generated command is the tested one.\n\n#### **2× DGX Spark (TP2)**\n\nTwo Sparks joined over ConnectX is the one layout the builder can't render, so\nit is spelled out here. Download the checkpoint on all nodes. Start the worker\nnode first and then the head node — the head coordinates the others, so they\nneed to be ready to connect when it comes up.\n\n```bash\n# COMMON variables example values — same on all nodes\nexport HEAD_IP=\"\" # The ConnectX IP of Node1 (Head Node)\nexport ETH_IF=\"enp1s0f1np1\" # CX7 ethernet interface which is UP\nexport IB_IF=\"rocep1s0f1,roceP2p1s0f1\" # BOTH matching RoCE interfaces\nexport MASTER_PORT=29501 # Free port on the Head Node\nexport CONTAINER_NAME=vllm_node\nexport IMAGE=\"vllm/vllm-openai:nightly\"\n```\n\n```bash\n# Head Node\ndocker run --privileged --ulimit nofile=1048576:1048576 --ipc=host \\\n --gpus all --rm --network host --name \"$CONTAINER_NAME\" --entrypoint=\"\" \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e VLLM_FLOAT32_MATMUL_PRECISION=high \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HOME/.cache/huggingface:/root/.cache/huggingface\" \\\n \"$IMAGE\" vllm serve nvidia/Qwen3.6-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend flashinfer \\\n --gpu-memory-utilization 0.5 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5,\"draft_tensor_parallel_size\":2}' \\\n --load-format fastsafetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_coder \\\n --enable-auto-tool-choice \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\"\n\n# Worker node\ndocker run --privileged --ulimit nofile=1048576:1048576 --ipc=host \\\n --gpus all --rm --network host --name \"$CONTAINER_NAME\" --entrypoint=\"\" \\\n -e MN_IF_NAME=\"$ETH_IF\" \\\n -e UCX_NET_DEVICES=\"$ETH_IF\" \\\n -e NCCL_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IB_HCA=\"$IB_IF\" \\\n -e NCCL_IB_DISABLE=0 \\\n -e OMPI_MCA_btl_tcp_if_include=\"$ETH_IF\" \\\n -e GLOO_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e TP_SOCKET_IFNAME=\"$ETH_IF\" \\\n -e NCCL_IGNORE_CPU_AFFINITY=1 \\\n -e VLLM_FLOAT32_MATMUL_PRECISION=high \\\n -e HEAD_IP=\"$HEAD_IP\" \\\n -e MASTER_PORT=\"$MASTER_PORT\" \\\n -v \"$HOME/.cache/huggingface:/root/.cache/huggingface\" \\\n \"$IMAGE\" vllm serve nvidia/Qwen3.6-27B-NVFP4 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --attention-backend flashinfer \\\n --gpu-memory-utilization 0.5 \\\n --max-model-len 262144 \\\n --max-num-seqs 8 \\\n --max-num-batched-tokens 8192 \\\n --enable-chunked-prefill \\\n --async-scheduling \\\n --enable-prefix-caching \\\n --speculative-config '{\"method\":\"mtp\",\"num_speculative_tokens\":5,\"draft_tensor_parallel_size\":2}' \\\n --load-format fastsafetensors \\\n --reasoning-parser qwen3 \\\n --tool-call-parser qwen3_coder \\\n --enable-auto-tool-choice \\\n --tensor-parallel-size 2 \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr \"$HEAD_IP\" \\\n --master-port \"$MASTER_PORT\" \\\n --headless\n```\n\n\n### AMD (MI300X, single GPU, BF16)\n\nBF16 fits one MI300X. Two settings matter on ROCm and neither is on by default:\n`VLLM_ROCM_USE_AITER=1` and an explicit attention backend. The auto-selected\n`ROCM_ATTN` cannot use its native kernel for this model and its decode slows\nsharply as the prompt grows; `ROCM_AITER_FA` stays flat.\n\n```bash\nVLLM_ROCM_USE_AITER=1 vllm serve Qwen/Qwen3.6-27B \\\n --attention-backend ROCM_AITER_FA \\\n --enable-auto-tool-choice --tool-call-parser qwen3_coder \\\n --reasoning-parser qwen3\n```\n\nMTP works with the same flags (add the **Speculative decoding** feature above).\nSingle-stream decode (tokens/s at ~200 / ~2k / ~12k / ~32k prompt tokens, 512\noutput tokens, prefix caching on, which is the default), measured 2026-10-12 on\none MI300X with `vllm/vllm-openai-rocm:v0.31.0`:\n\n| | MI300X |\n|---|---|\n| image defaults (`ROCM_ATTN`) | 66 / 56 / 27 / 13 |\n| AITER + `ROCM_AITER_UNIFIED_ATTN` | 66 / 66 / 60 / 49 |\n| AITER + `ROCM_AITER_FA` | 67 / 67 / 65 / 62 |\n| + MTP (`num_speculative_tokens: 3`) | 139 / 176 / 168 / 157 |\n\nThe synthetic prompts are repetitive, which flatters MTP; on real coding-agent\nrequests (10-25k-token contexts) decode averaged 27 tok/s with image defaults,\n68 with `ROCM_AITER_FA` and 161 with MTP added. GSM8K (first 250 test\nquestions, greedy, thinking off) is unchanged: 0.952 image defaults, 0.952\n`ROCM_AITER_FA`, 0.964 `ROCM_AITER_UNIFIED_ATTN`, 0.964 `ROCM_AITER_FA` + MTP.\n\n### Docker (Intel XPU B60 / B70)\n\nValidated on 4× Intel Arc Pro B60 or 2x Intel Arc Pro B70 (B60 24 GB, B70 32 GB per card) with the\nofficial vLLM XPU image `vllm/vllm-openai-xpu:latest`.\n\n```bash\ndocker run --device /dev/dri \\\n -v /dev/dri/by-path:/dev/dri/by-path --shm-size=16g \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-xpu:latest \\\n Qwen/Qwen3.6-27B \\\n --reasoning-parser qwen3 \\\n --tensor-parallel-size 4 \\\n --max-model-len 8192 \\\n --enforce-eager\n```\n\n## Processing Ultra-Long Texts\n\nQwen3.6-27B natively supports `262,144` tokens. To serve longer inputs, turn on the\n**Long Context** feature above: it applies the model card's YaRN RoPE\nconfiguration via `--hf-overrides` and raises `--max-model-len` to ~1,010,000 tokens.\nStatic YaRN holds the scaling factor constant regardless of input length, so\nleave it off unless you actually serve long prompts; for a ~524K workload,\nhalve the card's `factor` to `2.0`.\n\nSee the [model card](https://huggingface.co/Qwen/Qwen3.6-27B#processing-ultra-long-texts) for the full parameter reference.\n\n## Client Usage\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\nresp = client.chat.completions.create(\n model=\"Qwen/Qwen3.6-27B\",\n messages=[{\"role\": \"user\", \"content\": \"Write a haiku about gated delta networks.\"}],\n max_tokens=256,\n)\nprint(resp.choices[0].message.content)\n```\n\n## Troubleshooting\n\n- **CUDA graph / Mamba cache size error:** reduce `--max-cudagraph-capture-size`\n (default 512). See [vLLM PR #34571](https://github.com/vllm-project/vllm/pull/34571).\n- **Disable reasoning:** add `--default-chat-template-kwargs '{\"enable_thinking\": false}'`.\n- **Prefix Caching (Mamba):** currently experimental in \"align\" mode.\n\n## References\n\n- [Model card](https://huggingface.co/Qwen/Qwen3.6-27B)\n- [FP8 checkpoint](https://huggingface.co/Qwen/Qwen3.6-27B-FP8)\n- [GPTQ-Int4 checkpoint](https://huggingface.co/Qwen/Qwen3.6-27B-GPTQ-Int4)\n- [NVIDIA NVFP4 checkpoint](https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4)\n- [Ascend W8A8 checkpoint](https://www.modelscope.cn/models/Eco-Tech/Qwen3.6-27B-w8a8-mxfp8)\n- [Ascend A3 W8A8 checkpoint](https://www.modelscope.cn/models/Eco-Tech/Qwen3.6-27B-w8a8)\n- [vLLM Ascend tutorial](https://docs.vllm.ai/projects/ascend/en/latest/tutorials/models/Qwen3.5-27B-Qwen3.6-27B.html)\n- [Qwen3.6-35B-A3B recipe](../Qwen3.6-35B-A3B)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/Qwen/Qwen3.6-27B.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Qwen/Qwen3.6-27B.yaml", "overlays": [ { "source": "vllm-ascend", "repo": "vllm-project/vllm-ascend", "ref": "main", "sha": "02ac27a19cb7676304a9ca6985ba56ce29664536", "path": "recipes/overlays/Qwen/Qwen3.6-27B.yaml", "edit_url": "https://github.com/vllm-project/vllm-ascend/edit/main/recipes/overlays/Qwen/Qwen3.6-27B.yaml" } ] } }