{ "hf_id": "Qwen/Qwen3.6-35B-A3B", "meta": { "title": "Qwen3.6-35B-A3B", "slug": "qwen3.6-35b-a3b", "provider": "Qwen", "description": "Smaller Qwen3.6 multimodal MoE model (35B total / 3B active) with BF16, FP8, and NVIDIA NVFP4 variants", "date_added": "2026-04-16", "date_updated": "2026-09-14", "tasks": [ "multimodal", "text" ], "performance_headline": "Compact Qwen3.6 MoE with 3B active parameters — single-GPU FP8 or 2-4 GPU or 4x Intel Arc Pro B60/B70 BF16 serving", "related_recipes": [ "Qwen/Qwen3.5-397B-A17B" ], "hardware": { "dgx_spark_gb10": "verified", "h100": "verified", "h200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified", "rtx_pro_6000": "verified", "dgx_station_gb300": "verified", "rtx_5090": "verified", "arc_pro_b60": "verified", "arc_pro_b70": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_USE_RUST_FRONTEND": "1" }, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve Qwen/Qwen3.6-35B-A3B \\\n --trust-remote-code \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser qwen3 \\\n --mm-encoder-tp-mode data", "argv": [ "vllm", "serve", "Qwen/Qwen3.6-35B-A3B", "--trust-remote-code", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "qwen3", "--mm-encoder-tp-mode", "data" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n vllm/vllm-openai:latest Qwen/Qwen3.6-35B-A3B \\\n --trust-remote-code \\\n --tensor-parallel-size 1 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser qwen3 \\\n --mm-encoder-tp-mode data", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_USE_RUST_FRONTEND=1", "vllm/vllm-openai:latest", "Qwen/Qwen3.6-35B-A3B", "--trust-remote-code", "--tensor-parallel-size", "1", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "qwen3", "--mm-encoder-tp-mode", "data" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/Qwen/Qwen3.6-35B-A3B/strategies/single_node_tep.json", "single_node_dep": "/Qwen/Qwen3.6-35B-A3B/strategies/single_node_dep.json", "multi_node_tp": "/Qwen/Qwen3.6-35B-A3B/strategies/multi_node_tp.json", "multi_node_tp_pp": "/Qwen/Qwen3.6-35B-A3B/strategies/multi_node_tp_pp.json", "multi_node_dep": "/Qwen/Qwen3.6-35B-A3B/strategies/multi_node_dep.json", "multi_node_tep": "/Qwen/Qwen3.6-35B-A3B/strategies/multi_node_tep.json", "kv_store_centralized_mooncake": "/Qwen/Qwen3.6-35B-A3B/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/Qwen/Qwen3.6-35B-A3B/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/Qwen/Qwen3.6-35B-A3B/hw/h200.json", "h100": "/Qwen/Qwen3.6-35B-A3B/hw/h100.json", "b200": "/Qwen/Qwen3.6-35B-A3B/hw/b200.json", "gb200": "/Qwen/Qwen3.6-35B-A3B/hw/gb200.json", "b300": "/Qwen/Qwen3.6-35B-A3B/hw/b300.json", "gb300": "/Qwen/Qwen3.6-35B-A3B/hw/gb300.json", "dgx_station_gb300": "/Qwen/Qwen3.6-35B-A3B/hw/dgx_station_gb300.json", "dgx_spark_gb10": "/Qwen/Qwen3.6-35B-A3B/hw/dgx_spark_gb10.json", "rtx_pro_6000": "/Qwen/Qwen3.6-35B-A3B/hw/rtx_pro_6000.json", "rtx_5090": "/Qwen/Qwen3.6-35B-A3B/hw/rtx_5090.json", "mi300x": "/Qwen/Qwen3.6-35B-A3B/hw/mi300x.json", "mi325x": "/Qwen/Qwen3.6-35B-A3B/hw/mi325x.json", "mi350x": "/Qwen/Qwen3.6-35B-A3B/hw/mi350x.json", "mi355x": "/Qwen/Qwen3.6-35B-A3B/hw/mi355x.json", "arc_pro_b60": "/Qwen/Qwen3.6-35B-A3B/hw/arc_pro_b60.json", "arc_pro_b70": "/Qwen/Qwen3.6-35B-A3B/hw/arc_pro_b70.json" } }, "model": { "model_id": "Qwen/Qwen3.6-35B-A3B", "min_vllm_version": "0.17.0", "default_frontend": "rust", "architecture": "moe", "parameter_count": "35B", "active_parameters": "3B", "context_length": 262144, "base_args": [ "--trust-remote-code" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable automatic tool choice with Qwen3 Coder parser", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder" ] }, "reasoning": { "description": "Enable chain-of-thought reasoning with Qwen3 parser", "args": [ "--reasoning-parser", "qwen3" ] }, "spec_decoding": { "description": "Multi-token prediction speculative decoding for lower latency", "default_mode": "mtp", "modes": { "mtp": { "label": "MTP", "description": "Built-in Multi-Token Prediction, 3 draft tokens.", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3,\"moe_backend\":\"triton\"}" ], "hardware_overrides": { "rtx_pro_6000": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"flashinfer_cutlass\"}" ] }, "rtx_5090": { "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":4,\"moe_backend\":\"flashinfer_cutlass\"}" ] } } }, "dspark": { "label": "DSpark", "description": "DSpark drafting with an externally trained draft model; the served checkpoint is unchanged.", "variants": [ "default" ], "args": [ "--speculative-config", "{\"model\": \"RedHatAI/Qwen3.6-35B-A3B-speculator.dspark\", \"num_speculative_tokens\": 7, \"method\": \"dspark\"}" ] } } }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "encoder_parallel": { "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.", "args": [ "--mm-encoder-tp-mode", "data" ] }, "long_context": { "description": "YaRN RoPE scaling past the native 262,144-token window to ~1,010,000 tokens. Static YaRN holds the scaling factor constant regardless of input length, so leave this off unless you actually serve long prompts; for ~524K, halve the factor to 2.0.", "args": [ "--hf-overrides", "{\"text_config\": {\"rope_parameters\": {\"mrope_interleaved\": true, \"mrope_section\": [11, 11, 10], \"rope_type\": \"yarn\", \"rope_theta\": 10000000, \"partial_rotary_factor\": 0.25, \"factor\": 4.0, \"original_max_position_embeddings\": 262144}}}", "--max-model-len", "1010000" ], "env": { "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1" } } }, "opt_in_features": [ "spec_decoding", "text_only", "long_context" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 84, "description": "Full precision BF16 — fits on 1x H200, 2x H100, 4x Intel Arc Pro B60 / B70", "hardware_overrides": { "rtx_pro_6000": { "extra_args": [ "--max-num-seqs", "8" ] } } }, "fp8": { "model_id": "Qwen/Qwen3.6-35B-A3B-FP8", "precision": "fp8", "vram_minimum_gb": 42, "description": "Qwen official FP8 checkpoint — single-GPU serving", "hardware_overrides": { "rtx_pro_6000": { "extra_env": { "VLLM_USE_DEEP_GEMM": "0" } } }, "json": "/Qwen/Qwen3.6-35B-A3B-FP8.json" }, "nvfp4": { "model_id": "nvidia/Qwen3.6-35B-A3B-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 21, "tp": 1, "min_vllm_version": "0.28.0", "docker_image": "vllm/vllm-openai:v0.28.0", "description": "NVIDIA ModelOpt NVFP4 checkpoint for Blackwell GPUs, including DGX Spark", "hardware_overrides": { "dgx_spark_gb10": { "extra_args": [ "--kv-cache-dtype", "fp8", "--attention-backend", "flashinfer", "--moe-backend", "marlin", "--gpu-memory-utilization", "0.5", "--max-model-len", "262144", "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--enable-chunked-prefill", "--async-scheduling", "--enable-prefix-caching", "--load-format", "fastsafetensors" ] }, "dgx_station_gb300": { "extra_args": [ "--kv-cache-dtype", "fp8", "--attention-backend", "flashinfer", "--moe-backend", "marlin", "--gpu-memory-utilization", "0.92", "--max-model-len", "262144", "--max-num-batched-tokens", "8192", "--enable-chunked-prefill", "--async-scheduling", "--enable-prefix-caching", "--load-format", "fastsafetensors" ] }, "rtx_pro_6000": { "extra_args": [ "--quantization", "modelopt_fp4", "--kv-cache-dtype", "fp8", "--block-size", "128", "--moe-backend", "marlin", "--attention-backend", "flashinfer", "--attention-config", "{\"use_trtllm_attention\":true}", "--gpu-memory-utilization", "0.9", "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--async-scheduling", "--enable-chunked-prefill", "--enable-prefix-caching", "--skip-mm-profiling", "--load-format", "fastsafetensors" ], "extra_env": { "VLLM_HAS_FLASHINFER_CUBIN": "1" } }, "rtx_5090": { "extra_args": [ "--quantization", "modelopt_fp4", "--kv-cache-dtype", "fp8", "--block-size", "128", "--moe-backend", "marlin", "--attention-backend", "flashinfer", "--attention-config", "{\"use_trtllm_attention\":true}", "--gpu-memory-utilization", "0.85", "--max-model-len", "65536", "--max-num-seqs", "8", "--max-num-batched-tokens", "8192", "--enable-chunked-prefill", "--enable-prefix-caching", "--async-scheduling", "--skip-mm-profiling" ], "extra_env": { "VLLM_HAS_FLASHINFER_CUBIN": "1" } } }, "json": "/nvidia/Qwen3.6-35B-A3B-NVFP4.json" }, "redhat_nvfp4": { "model_id": "RedHatAI/Qwen3.6-35B-A3B-NVFP4", "precision": "nvfp4", "label": "NVFP4 (RedHat)", "vram_minimum_gb": 21, "description": "RedHatAI compressed-tensors NVFP4 quantization for Blackwell GPUs", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Qwen3.6-35B-A3B-NVFP4.json" }, "redhat_fp8": { "model_id": "RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic", "precision": "fp8", "label": "FP8 Dynamic (RedHat)", "vram_minimum_gb": 42, "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Qwen3.6-35B-A3B-FP8-dynamic.json" }, "redhat_fp8_plain": { "model_id": "RedHatAI/Qwen3.6-35B-A3B-FP8", "precision": "fp8", "label": "FP8 (RedHat)", "vram_minimum_gb": 42, "description": "RedHatAI FP8 quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Qwen3.6-35B-A3B-FP8.json" } }, "hardware_overrides": { "amd": { "extra_args": [], "extra_env": { "VLLM_ROCM_USE_AITER": "1" } }, "xpu": { "extra_args": [ "--enforce-eager" ] } }, "guide": "## Overview\n\n[Qwen3.6-35B-A3B](https://huggingface.co/Qwen/Qwen3.6-35B-A3B) is the smaller sibling\nof Qwen3.5, sharing the same gated-delta-networks MoE architecture but with 35B total\nparameters and 3B activated (256 experts, 8 routed + 1 shared). The recipe covers the\nBF16 base model, Qwen's official FP8 checkpoint, and NVIDIA's ModelOpt NVFP4\ncheckpoint.\n\n## Prerequisites\n\n- **vLLM version:** >= 0.17.0\n- **NVFP4 vLLM version:** >= 0.28.0. 0.24.0 loads the checkpoint, but\n FlashInfer only selects its XQA decode kernel on SM120 GPUs from 0.28.0.\n- **Hardware (BF16):** 1x H200, 2x H100, 4x Intel Arc Pro B60 / B70\n- **Hardware (FP8):** single H100/H200 or 1x MI300X/MI325X/MI355X\n- **Hardware (NVFP4):** NVIDIA Blackwell GPUs, including DGX Spark (GB10)\n\n## NVFP4 on Blackwell\n\nAll NVFP4 tuning lives in `variants.nvfp4.hardware_overrides`, keyed by GPU\nprofile, so the exact flag set for a box is readable straight from the recipe\nYAML (or from `by_hardware` in the JSON API) without going through the\ncommand builder.\n\nFour profiles are tuned and verified. DGX Spark (GB10) and DGX Station\n(GB300) run the checkpoint on the default loader with FlashInfer attention,\n`--moe-backend marlin` and a 3-token MTP draft on Triton; Spark shares\nunified memory with the host, so `--gpu-memory-utilization` is capped at 0.5.\nThe SM120 parts (RTX PRO 6000, RTX 5090) load through `--quantization\nmodelopt_fp4` with FlashInfer's TRT-LLM attention kernels, which need\n`--block-size 128` and `VLLM_HAS_FLASHINFER_CUBIN=1`, and sustain a deeper\n4-token MTP draft on `flashinfer_cutlass`. The RTX 5090's 32 GB caps context\nat 64K.\n\nDatacenter Blackwell (B200/GB200/B300/GB300) intentionally carries no\noverrides — it runs vLLM's own NVFP4 defaults, which select the native FP4\npath rather than the Marlin W4A16 kernel the workstation parts need.\n\n### Measured on DGX Spark (GB10)\n\nThe GB10 profile above was re-measured on 2026-08-31 with aiperf 0.12.0 on\nvLLM 0.28.0 (Ubuntu 24.04.4, driver 580.159.03, CUDA 13.0), one GB10 at TP1,\nserving from `vllm/vllm-openai:v0.28.0`. On SPEED-Bench at 8K input / 256\noutput tokens and concurrency 1, it sustained 6,265 prefill tok/s and 97.7\ndecode tok/s at a 3.95 s mean end-to-end latency. The flag set is unchanged —\nthe sweep confirms the tuning rather than revising it, so the command the\nbuilder emits for GB10 is the one that produced these numbers.\n\n### Docker (Intel XPU B60 / B70)\n\nValidated on 4× Intel Arc Pro B60 / B70 (B60 24 GB, B70 32 GB per card) with the\nofficial vLLM XPU image `vllm/vllm-openai-xpu:latest`.\n\n```bash\ndocker run --device /dev/dri \\\n -v /dev/dri/by-path:/dev/dri/by-path --shm-size=16g \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai-xpu:latest \\\n Qwen/Qwen3.6-35B-A3B \\\n --reasoning-parser qwen3 \\\n --tensor-parallel-size 4 \\\n --max-model-len 8192 \\\n --enforce-eager\n```\n\n## Processing Ultra-Long Texts\n\nQwen3.6-35B-A3B natively supports `262,144` tokens. To serve longer inputs, turn on the\n**Long Context** feature above: it applies the model card's YaRN RoPE\nconfiguration via `--hf-overrides` and raises `--max-model-len` to ~1,010,000 tokens.\nStatic YaRN holds the scaling factor constant regardless of input length, so\nleave it off unless you actually serve long prompts; for a ~524K workload,\nhalve the card's `factor` to `2.0`.\n\nSee the [model card](https://huggingface.co/Qwen/Qwen3.6-35B-A3B#processing-ultra-long-texts) for the full parameter reference.\n\n## Client Usage\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\")\nresp = client.chat.completions.create(\n model=\"Qwen/Qwen3.6-35B-A3B\",\n messages=[{\"role\": \"user\", \"content\": \"Explain gated delta networks in one paragraph.\"}],\n max_tokens=512,\n)\nprint(resp.choices[0].message.content)\n```\n\n## Troubleshooting\n\n- **CUDA graph / Mamba cache size error:** reduce `--max-cudagraph-capture-size`\n (default 512). See [vLLM PR #34571](https://github.com/vllm-project/vllm/pull/34571).\n- **Reasoning disable:** add `--default-chat-template-kwargs '{\"enable_thinking\": false}'`.\n- **Prefix Caching (Mamba):** currently experimental in \"align\" mode.\n\n## References\n\n- [Qwen3.6-35B-A3B on Hugging Face](https://huggingface.co/Qwen/Qwen3.6-35B-A3B)\n- [FP8 checkpoint](https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8)\n- [NVIDIA NVFP4 checkpoint](https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4)\n- [Qwen3.5 recipe (sibling 397B-A17B flagship)](../Qwen3.5-397B-A17B)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/Qwen/Qwen3.6-35B-A3B.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Qwen/Qwen3.6-35B-A3B.yaml", "overlays": [] } }