{ "hf_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "meta": { "title": "Qwen3-235B-A22B-Instruct", "slug": "qwen3-235b-a22b-instruct-2507", "provider": "Qwen", "description": "Flagship Qwen3 MoE instruct model with 235B total and 22B active parameters, tuned for high-quality text generation.", "date_added": "2025-10-15", "date_updated": "2026-04-17", "tasks": [ "text" ], "performance_headline": "Verified on 4x/8x H200, MI300X/MI325X/MI355X nodes (BF16 and FP8)", "related_recipes": [], "hardware": { "h200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve Qwen/Qwen3-235B-A22B-Instruct-2507 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes", "argv": [ "vllm", "serve", "Qwen/Qwen3-235B-A22B-Instruct-2507", "--trust-remote-code", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "hermes" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest Qwen/Qwen3-235B-A22B-Instruct-2507 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser hermes", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "Qwen/Qwen3-235B-A22B-Instruct-2507", "--trust-remote-code", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "hermes" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/Qwen/Qwen3-235B-A22B-Instruct-2507/strategies/single_node_tep.json", "single_node_dep": "/Qwen/Qwen3-235B-A22B-Instruct-2507/strategies/single_node_dep.json", "multi_node_tp": "/Qwen/Qwen3-235B-A22B-Instruct-2507/strategies/multi_node_tp.json", "multi_node_tp_pp": "/Qwen/Qwen3-235B-A22B-Instruct-2507/strategies/multi_node_tp_pp.json", "multi_node_dep": "/Qwen/Qwen3-235B-A22B-Instruct-2507/strategies/multi_node_dep.json", "multi_node_tep": "/Qwen/Qwen3-235B-A22B-Instruct-2507/strategies/multi_node_tep.json", "pd_cluster": "/Qwen/Qwen3-235B-A22B-Instruct-2507/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/Qwen/Qwen3-235B-A22B-Instruct-2507/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/Qwen/Qwen3-235B-A22B-Instruct-2507/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/h200.json", "h100": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/h100.json", "b200": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/b200.json", "gb200": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/gb200.json", "b300": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/b300.json", "gb300": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/gb300.json", "mi300x": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/mi300x.json", "mi325x": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/mi325x.json", "mi350x": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/mi350x.json", "mi355x": "/Qwen/Qwen3-235B-A22B-Instruct-2507/hw/mi355x.json" } }, "model": { "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "min_vllm_version": "0.10.0", "architecture": "moe", "parameter_count": "235B", "active_parameters": "22B", "context_length": 262144, "base_args": [ "--trust-remote-code" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable automatic tool choice with Hermes-compatible parser", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "hermes" ] }, "spec_decoding": { "description": "Eagle3 speculative decoding with externally trained draft model; the served checkpoint is unchanged.", "args": [ "--speculative-config", "{\"model\": \"RedHatAI/Qwen3-235B-A22B-Instruct-2507-speculator.eagle3\", \"num_speculative_tokens\": 3, \"method\": \"eagle3\"}" ] } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 564, "description": "Full precision BF16 — requires 4x H200 or 8x MI300X/MI325X/MI355X" }, "fp8": { "model_id": "Qwen/Qwen3-235B-A22B-FP8", "precision": "fp8", "vram_minimum_gb": 240, "tp": 4, "description": "Qwen official FP8 checkpoint for improved efficiency on SM90+", "json": "/Qwen/Qwen3-235B-A22B-FP8.json" }, "nvfp4": { "model_id": "nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 141, "description": "NVIDIA NVFP4 quantized weights for Blackwell GPUs", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/nvidia/Qwen3-235B-A22B-Instruct-2507-NVFP4.json" }, "redhat_nvfp4": { "model_id": "RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4", "precision": "nvfp4", "label": "NVFP4 (RedHat)", "vram_minimum_gb": 152, "description": "RedHatAI NVFP4 quantization", "json": "/RedHatAI/Qwen3-235B-A22B-Instruct-2507-NVFP4.json" } }, "kv_offload_support": { "offloading_cpu": "verified", "offloading_fs": "verified" }, "hardware_overrides": {}, "guide": "## Overview\n\n[Qwen3](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507) is the flagship instruct MoE model in the Qwen3 series, with 235B total parameters and 22B active parameters. This guide covers deploying the model efficiently with vLLM on NVIDIA and AMD GPUs.\n\n## Prerequisites\n\n### NVIDIA CUDA (pip)\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend=auto\n```\n\n### AMD ROCm (pip)\n> Note: The vLLM wheel for ROCm requires Python 3.12, ROCm 7.0, and glibc >= 2.35. Use the Docker flow if your environment is incompatible.\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/0.14.1/rocm700\n```\n\n## Deployment Configurations\n\n### BF16 on MI300X/MI325X/MI355X (4 GPUs)\n\n```bash\nHIP_VISIBLE_DEVICES=\"4,5,6,7\" \\\nVLLM_ROCM_USE_AITER=1 \\\nVLLM_ROCM_USE_AITER_MHA=0 \\\nSAFETENSORS_FAST_GPU=1 \\\nvllm serve Qwen/Qwen3-235B-A22B \\\n --trust-remote-code \\\n -tp 4 \\\n --swap-space 32 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 32768 \\\n --max-model-len 32768 \\\n --no-enable-prefix-caching \\\n --gpu-memory-utilization 0.8\n```\n\n### FP8 on MI300X/MI325X/MI355X (4 GPUs)\n\n```bash\nHIP_VISIBLE_DEVICES=\"4,5,6,7\" \\\nVLLM_ROCM_USE_AITER=1 \\\nVLLM_ROCM_USE_AITER_MHA=0 \\\nSAFETENSORS_FAST_GPU=1 \\\nvllm serve Qwen/Qwen3-235B-A22B-FP8 \\\n --trust-remote-code \\\n -tp 4 \\\n --swap-space 16 \\\n --distributed-executor-backend mp \\\n --max-num-batched-tokens 32768 \\\n --max-model-len 32768 \\\n --no-enable-prefix-caching \\\n --gpu-memory-utilization 0.8\n```\n\n### TPU Deployment\n\n- [Qwen3-32B on Trillium (v6e)](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Qwen3)\n- [Qwen3-4B on Trillium (v6e)](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Qwen3)\n\n## Client Usage\n\n```bash\nvllm bench serve \\\n --model \"Qwen/Qwen3-235B-A22B-FP8\" \\\n --dataset-name random \\\n --random-input-len 8192 \\\n --random-output-len 1024 \\\n --request-rate 10000 \\\n --num-prompts 16 \\\n --ignore-eos \\\n --trust-remote-code\n```\n\n## Troubleshooting\n\n- Use `--max-num-batched-tokens` and `--max-model-len` to fit memory constraints on smaller nodes.\n- If OOM at startup, lower `--gpu-memory-utilization` (e.g. 0.8) and disable prefix caching.\n- AMD builds require `VLLM_ROCM_USE_AITER=1` for best performance.\n\n## References\n\n- [Model card](https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507)\n- [FP8 checkpoint](https://huggingface.co/Qwen/Qwen3-235B-A22B-FP8)\n- [vLLM documentation](https://docs.vllm.ai/)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/Qwen/Qwen3-235B-A22B-Instruct-2507.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Qwen/Qwen3-235B-A22B-Instruct-2507.yaml", "overlays": [] } }