{ "hf_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "meta": { "title": "Qwen3-VL-235B-A22B-Instruct", "slug": "qwen3-vl-235b-a22b-instruct", "provider": "Qwen", "description": "Qwen3-VL flagship MoE vision-language model with 235B total / 22B active parameters, supporting images, video, and long context.", "date_added": "2025-09-28", "date_updated": "2026-04-17", "tasks": [ "multimodal", "text" ], "performance_headline": "Strong on images, video, and text — #1 open model on text on lmarena.ai at release", "related_recipes": [], "hardware": { "h100": "verified", "h200": "verified", "b200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve Qwen/Qwen3-VL-235B-A22B-Instruct \\\n --tensor-parallel-size 8 \\\n --async-scheduling \\\n --mm-encoder-tp-mode data", "argv": [ "vllm", "serve", "Qwen/Qwen3-VL-235B-A22B-Instruct", "--tensor-parallel-size", "8", "--async-scheduling", "--mm-encoder-tp-mode", "data" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest Qwen/Qwen3-VL-235B-A22B-Instruct \\\n --tensor-parallel-size 8 \\\n --async-scheduling \\\n --mm-encoder-tp-mode data", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "Qwen/Qwen3-VL-235B-A22B-Instruct", "--tensor-parallel-size", "8", "--async-scheduling", "--mm-encoder-tp-mode", "data" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/Qwen/Qwen3-VL-235B-A22B-Instruct/strategies/single_node_tep.json", "single_node_dep": "/Qwen/Qwen3-VL-235B-A22B-Instruct/strategies/single_node_dep.json", "multi_node_tp": "/Qwen/Qwen3-VL-235B-A22B-Instruct/strategies/multi_node_tp.json", "multi_node_tp_pp": "/Qwen/Qwen3-VL-235B-A22B-Instruct/strategies/multi_node_tp_pp.json", "multi_node_dep": "/Qwen/Qwen3-VL-235B-A22B-Instruct/strategies/multi_node_dep.json", "multi_node_tep": "/Qwen/Qwen3-VL-235B-A22B-Instruct/strategies/multi_node_tep.json", "pd_cluster": "/Qwen/Qwen3-VL-235B-A22B-Instruct/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/Qwen/Qwen3-VL-235B-A22B-Instruct/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/Qwen/Qwen3-VL-235B-A22B-Instruct/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/h200.json", "h100": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/h100.json", "b200": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/b200.json", "gb200": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/gb200.json", "b300": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/b300.json", "gb300": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/gb300.json", "mi300x": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/mi300x.json", "mi325x": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/mi325x.json", "mi350x": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/mi350x.json", "mi355x": "/Qwen/Qwen3-VL-235B-A22B-Instruct/hw/mi355x.json" } }, "model": { "model_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "min_vllm_version": "0.11.0", "architecture": "moe", "parameter_count": "235B", "active_parameters": "22B", "context_length": 262144, "base_args": [], "base_env": {}, "docker_image": { "amd": "vllm/vllm-openai-rocm:v0.28.0" }, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" }, "extras": [ { "note": "Recommended for offline multimodal inference (image/video pre-processing helpers)", "command": "uv pip install qwen-vl-utils==0.0.14", "optional": true } ] } }, "features": { "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "encoder_parallel": { "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.", "args": [ "--mm-encoder-tp-mode", "data" ] }, "long_context": { "description": "Extend the 262K context to 1M tokens with YaRN. Static scaling costs quality on short prompts and the wider window inflates the KV budget, so it is opt-in.", "args": [ "--hf-overrides", "{\"text_config\":{\"rope_parameters\":{\"rope_type\":\"yarn\",\"factor\":3.0,\"original_max_position_embeddings\":262144,\"mrope_section\":[24,20,20],\"mrope_interleaved\":true}}}", "--max-model-len", "1000000" ], "env": { "VLLM_ALLOW_LONG_MAX_MODEL_LEN": "1" } } }, "opt_in_features": [ "text_only", "long_context" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 564, "description": "Full BF16 — ideal on H200/B200 with 8 GPUs" }, "fp8": { "model_id": "Qwen/Qwen3-VL-235B-A22B-Instruct-FP8", "precision": "fp8", "vram_minimum_gb": 282, "tp": 4, "description": "Qwen official FP8 checkpoint for optimal H100 memory efficiency", "json": "/Qwen/Qwen3-VL-235B-A22B-Instruct-FP8.json" }, "nvfp4": { "model_id": "nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 141, "description": "NVIDIA NVFP4 quantized weights for Blackwell GPUs", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/nvidia/Qwen3-VL-235B-A22B-Instruct-NVFP4.json" }, "redhat_fp8": { "model_id": "RedHatAI/Qwen3-VL-235B-A22B-Instruct-FP8-dynamic", "precision": "fp8", "label": "FP8 Dynamic (RedHat)", "vram_minimum_gb": 282, "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Qwen3-VL-235B-A22B-Instruct-FP8-dynamic.json" }, "redhat_fp8_block": { "model_id": "RedHatAI/Qwen3-VL-235B-A22B-Instruct-FP8-block", "precision": "fp8", "label": "FP8 Block (RedHat)", "vram_minimum_gb": 282, "description": "RedHatAI FP8 block quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Qwen3-VL-235B-A22B-Instruct-FP8-block.json" }, "redhat_nvfp4": { "model_id": "RedHatAI/Qwen3-VL-235B-A22B-Instruct-NVFP4", "precision": "nvfp4", "label": "NVFP4 (RedHat)", "vram_minimum_gb": 150, "description": "RedHatAI NVFP4 quantization", "json": "/RedHatAI/Qwen3-VL-235B-A22B-Instruct-NVFP4.json" } }, "kv_offload_support": { "offloading_cpu": "verified", "offloading_fs": "verified" }, "hardware_overrides": { "hopper": { "extra_args": [ "--mm-encoder-tp-mode", "data", "--async-scheduling" ], "extra_env": {} }, "blackwell": { "extra_args": [ "--mm-encoder-tp-mode", "data", "--async-scheduling" ], "extra_env": {} }, "amd": { "extra_args": [ "--mm-encoder-tp-mode", "data", "--async-scheduling", "--enable-chunked-prefill" ], "extra_env": { "VLLM_WORKER_MULTIPROC_METHOD": "spawn", "HIP_FORCE_DEV_KERNARG": "1", "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT": "1", "SAFETENSORS_FAST_GPU": "1" } } }, "guide": "## Overview\n\n[Qwen3-VL](https://github.com/QwenLM/Qwen3-VL) is the most powerful vision-language model in the Qwen series, delivering upgrades to text understanding & generation, visual perception & reasoning, extended context, spatial/video dynamics, and agent interaction. The flagship `Qwen3-VL-235B-A22B-Instruct` is a MoE model that requires at least 8 GPUs with ≥80 GB memory each (A100/H100/H200 class).\n\n## Prerequisites\n\n```bash\nuv venv\nsource .venv/bin/activate\n\n# Install vLLM >= 0.11.0\nuv pip install -U vllm\n\n# Install Qwen-VL utility library (recommended for offline inference)\nuv pip install qwen-vl-utils==0.0.14\n```\n\n## Deployment Configurations\n\n### H100 (Image + Video, FP8)\n```bash\nvllm serve Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 \\\n --tensor-parallel-size 8 \\\n --mm-encoder-tp-mode data \\\n --enable-expert-parallel \\\n --async-scheduling\n```\n\n### H100 (Image-Only, FP8, TP4)\n```bash\nvllm serve Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 \\\n --tensor-parallel-size 4 \\\n --limit-mm-per-prompt.video 0 \\\n --async-scheduling \\\n --gpu-memory-utilization 0.95 \\\n --max-num-seqs 128\n```\n\n### A100 & H100 (Image-Only, BF16)\n```bash\nvllm serve Qwen/Qwen3-VL-235B-A22B-Instruct \\\n --tensor-parallel-size 8 \\\n --limit-mm-per-prompt.video 0 \\\n --async-scheduling\n```\n\n### A100 & H100 (Image + Video, BF16)\n```bash\nvllm serve Qwen/Qwen3-VL-235B-A22B-Instruct \\\n --tensor-parallel-size 8 \\\n --max-model-len 128000 \\\n --async-scheduling\n```\n\n### H200 & B200\n```bash\nvllm serve Qwen/Qwen3-VL-235B-A22B-Instruct \\\n --tensor-parallel-size 8 \\\n --mm-encoder-tp-mode data \\\n --async-scheduling\n```\n\n### MI300X/MI325X/MI355X (BF16, TP8)\n```bash\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport HIP_FORCE_DEV_KERNARG=\"1\"\nexport HIP_VISIBLE_DEVICES=\"0,1,2,3,4,5,6,7\"\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_ROCM_USE_AITER=\"1\"\nexport VLLM_ROCM_USE_AITER_MHA=\"1\"\nexport VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=\"1\"\nvllm serve Qwen/Qwen3-VL-235B-A22B-Instruct \\\n --tensor-parallel-size 8 \\\n --mm-encoder-tp-mode data \\\n --enable-expert-parallel \\\n --async-scheduling \\\n --gpu-memory-utilization 0.94 \\\n --max-model-len 32768 \\\n --max-num-seqs 10240 \\\n --max-num-batched-tokens 32768 \\\n --attention-backend ROCM_AITER_FA\n```\n\n### MI300X/MI325X/MI355X (FP8, TP8)\n```bash\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport HIP_FORCE_DEV_KERNARG=\"1\"\nexport HIP_VISIBLE_DEVICES=\"0,1,2,3,4,5,6,7\"\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_ROCM_USE_AITER=\"1\"\nexport VLLM_ROCM_USE_AITER_MHA=\"1\"\nexport VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=\"1\"\nvllm serve Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 \\\n --tensor-parallel-size 8 \\\n --mm-encoder-tp-mode data \\\n --enable-expert-parallel \\\n --async-scheduling \\\n --enable-chunked-prefill \\\n --limit-mm-per-prompt '{\"image\": 1}' \\\n --gpu-memory-utilization 0.94 \\\n --max-model-len 32768 \\\n --max-num-seqs 10240 \\\n --max-num-batched-tokens 32768 \\\n --kv-cache-dtype fp8 \\\n --compilation-config '{\"mode\": 3, \"cudagraph_mode\": \"FULL_AND_PIECEWISE\", \"custom_ops\": [\"+rms_norm\", \"+quant_fp8\"]}' \\\n --attention-backend ROCM_AITER_FA\n```\n\n> The above configuration runs efficiently at high concurrencies (up to 512) for context lengths up to 8k. If TPOT is not satisfactory, lower `--max-num-batched-tokens`. For longer contexts, `--max-num-seqs` and `--max-num-batched-tokens` may need retuning. \n\n### MI300X / MI355X (FP8, EAGLE3 Speculative Decoding)\n\nAdds EAGLE3 speculative decoding (`RedHatAI/Qwen3-VL-235B-A22B-Instruct-speculator.eagle3`) on top of the FP8 target. **`--attention-backend ROCM_AITER_FA` is mandatory** — the default `ROCM_ATTN` path makes SD slower than baseline on ROCm. Keep `VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=0` and omit `--async-scheduling` (incompatible with SD).\n\n```bash\nexport SAFETENSORS_FAST_GPU=\"1\"\nexport HIP_FORCE_DEV_KERNARG=\"1\"\nexport HIP_VISIBLE_DEVICES=\"0,1,2,3,4,5,6,7\"\nexport VLLM_WORKER_MULTIPROC_METHOD=\"spawn\"\nexport VLLM_ROCM_USE_AITER=\"1\"\nexport VLLM_ROCM_USE_AITER_MHA=\"1\"\nexport VLLM_ROCM_USE_AITER_RMSNORM=\"0\"\nexport VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT=\"0\"\n\nvllm serve Qwen/Qwen3-VL-235B-A22B-Instruct-FP8 \\\n --tensor-parallel-size 8 \\\n --gpu-memory-utilization 0.94 \\\n --distributed-executor-backend mp \\\n --enable-chunked-prefill \\\n --max-model-len 16384 \\\n --max-num-seqs 32 \\\n --max-num-batched-tokens 8192 \\\n --mm-encoder-tp-mode data \\\n --enable-expert-parallel \\\n --attention-backend ROCM_AITER_FA \\\n --compilation-config '{\"mode\": 3, \"cudagraph_mode\": \"FULL_AND_PIECEWISE\", \"custom_ops\": [\"-rms_norm\"], \"pass_config\": {\"fuse_norm_quant\":false}}' \\\n --speculative-config '{\"model\": \"RedHatAI/Qwen3-VL-235B-A22B-Instruct-speculator.eagle3\", \"method\": \"eagle3\", \"num_speculative_tokens\": 4}'\n```\n\n## Configuration Tips\n\n- Use `--limit-mm-per-prompt.video 0` if your server only serves image inputs to save memory.\n- `OMP_NUM_THREADS=1` reduces CPU contention during preprocessing.\n- The model's context length is 262K. Reduce `--max-model-len` (e.g. 128000) if you don't need the full range.\n- `--async-scheduling` overlaps scheduling with decoding for better throughput.\n- `--mm-encoder-tp-mode data` deploys the vision encoder in data-parallel fashion for better performance.\n- If your inputs are mostly unique, pass `--mm-processor-cache-gb 0` to skip caching overhead.\n- Extend context to 1M tokens with the **Long Context** toggle, which applies YaRN via\n `--hf-overrides` and raises `--max-model-len`. Static YaRN costs quality on short prompts.\n\nText-only mode: pass `--limit-mm-per-prompt.video 0 --limit-mm-per-prompt.image 0` to free memory for KV cache when serving text-only traffic.\n\n## Client Usage\n\n```python\nimport time\nfrom openai import OpenAI\n\nclient = OpenAI(api_key=\"EMPTY\", base_url=\"http://localhost:8000/v1\", timeout=3600)\n\nmessages = [{\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"https://ofasys-multimodal-wlcb-3-toshanghai.oss-accelerate.aliyuncs.com/wpf272043/keepme/image/receipt.png\"}},\n {\"type\": \"text\", \"text\": \"Read all the text in the image.\"},\n ],\n}]\n\nstart = time.time()\nresponse = client.chat.completions.create(\n model=\"Qwen/Qwen3-VL-235B-A22B-Instruct\",\n messages=messages,\n max_tokens=2048,\n)\nprint(f\"Response costs: {time.time() - start:.2f}s\")\nprint(response.choices[0].message.content)\n```\n\n## Troubleshooting\n\n- OOM on A100 / H100 BF16: reduce `--max-model-len`, drop to image-only, or switch to the FP8 checkpoint.\n- If enabling `--mm-encoder-tp-mode data` raises memory pressure, lower `--gpu-memory-utilization`.\n\n## References\n\n- [Model card](https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct)\n- [FP8 checkpoint](https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct-FP8)\n- [Qwen3-VL GitHub](https://github.com/QwenLM/Qwen3-VL)\n- [vLLM multimodal inputs guide](https://docs.vllm.ai/en/latest/features/multimodal_inputs.html)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/Qwen/Qwen3-VL-235B-A22B-Instruct.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Qwen/Qwen3-VL-235B-A22B-Instruct.yaml", "overlays": [] } }