{
  "hf_id": "Qwen/Qwen2.5-VL-72B-Instruct",
  "meta": {
    "title": "Qwen2.5-VL-72B-Instruct",
    "slug": "qwen2.5-vl-72b-instruct",
    "provider": "Qwen",
    "description": "Qwen2.5-VL dense vision-language model (72B) for high-quality image and video understanding.",
    "date_added": "2025-08-22",
    "date_updated": "2026-04-17",
    "tasks": [
      "multimodal",
      "text"
    ],
    "performance_headline": "Verified on 4x A100 and 4x MI300X/MI325X/MI355X with BF16",
    "related_recipes": [
      "Qwen/Qwen2.5-VL-7B-Instruct"
    ],
    "hardware": {
      "mi300x": "verified",
      "mi325x": "verified",
      "mi355x": "verified"
    }
  },
  "recommended_command": {
    "hardware": "h200",
    "strategy": "single_node_tp",
    "variant": "default",
    "node_count": 1,
    "deploy_type": "single_node",
    "env": {},
    "docker_image": "vllm/vllm-openai:latest",
    "command": "vllm serve Qwen/Qwen2.5-VL-72B-Instruct \\\n  --tensor-parallel-size 8 \\\n  --mm-encoder-tp-mode data",
    "argv": [
      "vllm",
      "serve",
      "Qwen/Qwen2.5-VL-72B-Instruct",
      "--tensor-parallel-size",
      "8",
      "--mm-encoder-tp-mode",
      "data"
    ],
    "docker_command": "docker run --gpus all \\\n  --privileged --ipc=host -p 8000:8000 \\\n  -v ~/.cache/huggingface:/root/.cache/huggingface \\\n  vllm/vllm-openai:latest Qwen/Qwen2.5-VL-72B-Instruct \\\n  --tensor-parallel-size 8 \\\n  --mm-encoder-tp-mode data",
    "docker_argv": [
      "docker",
      "run",
      "--gpus",
      "all",
      "--privileged",
      "--ipc=host",
      "-p",
      "8000:8000",
      "-v",
      "~/.cache/huggingface:/root/.cache/huggingface",
      "vllm/vllm-openai:latest",
      "Qwen/Qwen2.5-VL-72B-Instruct",
      "--tensor-parallel-size",
      "8",
      "--mm-encoder-tp-mode",
      "data"
    ],
    "strategy_spec": {
      "name": "single_node_tp",
      "deploy_type": "single_node",
      "display_name": "Tensor Parallel",
      "short_label": "TP",
      "orientation": "latency",
      "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n",
      "hardware_match": {
        "min_gpus": 1,
        "max_gpus": 8,
        "multi_node": false
      },
      "vllm_args": [],
      "parallel_flag": "--tensor-parallel-size"
    },
    "hardware_profile": {
      "brand": "NVIDIA",
      "generation": "hopper",
      "compute_arch": "sm90",
      "display_name": "H200",
      "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node",
      "gpu_count": 8,
      "vram_gb": 1128,
      "multi_node": false
    },
    "alternatives": {
      "multi_node_tp": "/Qwen/Qwen2.5-VL-72B-Instruct/strategies/multi_node_tp.json",
      "multi_node_tp_pp": "/Qwen/Qwen2.5-VL-72B-Instruct/strategies/multi_node_tp_pp.json",
      "kv_store_centralized_mooncake": "/Qwen/Qwen2.5-VL-72B-Instruct/strategies/kv_store_centralized_mooncake.json",
      "kv_store_distributed_mooncake": "/Qwen/Qwen2.5-VL-72B-Instruct/strategies/kv_store_distributed_mooncake.json"
    },
    "by_hardware": {
      "h200": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/h200.json",
      "h100": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/h100.json",
      "b200": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/b200.json",
      "gb200": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/gb200.json",
      "b300": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/b300.json",
      "gb300": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/gb300.json",
      "mi300x": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/mi300x.json",
      "mi325x": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/mi325x.json",
      "mi350x": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/mi350x.json",
      "mi355x": "/Qwen/Qwen2.5-VL-72B-Instruct/hw/mi355x.json"
    }
  },
  "model": {
    "model_id": "Qwen/Qwen2.5-VL-72B-Instruct",
    "min_vllm_version": "0.10.2",
    "architecture": "dense",
    "parameter_count": "72B",
    "active_parameters": "72B",
    "context_length": 128000,
    "base_args": [],
    "base_env": {},
    "install": {
      "pip": {
        "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto"
      },
      "docker": {
        "command": "docker pull vllm/vllm-openai:latest"
      }
    }
  },
  "features": {
    "text_only": {
      "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.",
      "args": [
        "--language-model-only"
      ]
    },
    "encoder_parallel": {
      "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.",
      "args": [
        "--mm-encoder-tp-mode",
        "data"
      ]
    }
  },
  "opt_in_features": [
    "text_only"
  ],
  "variants": {
    "default": {
      "precision": "bf16",
      "vram_minimum_gb": 173,
      "description": "Full precision BF16 — 4x A100 80GB or 4x MI300X/MI325X/MI355X"
    },
    "awq": {
      "model_id": "Qwen/Qwen2.5-VL-72B-Instruct-AWQ",
      "precision": "int4",
      "vram_minimum_gb": 43,
      "description": "AWQ 4-bit quantized weights",
      "extra_args": [
        "--quantization",
        "awq"
      ],
      "json": "/Qwen/Qwen2.5-VL-72B-Instruct-AWQ.json"
    },
    "fp8": {
      "model_id": "RedHatAI/Qwen2.5-VL-72B-Instruct-FP8-dynamic",
      "precision": "fp8",
      "vram_minimum_gb": 87,
      "description": "RedHatAI FP8 (E4M3) with dynamic per-token activation quantization",
      "extra_args": [
        "--kv-cache-dtype",
        "fp8"
      ],
      "json": "/RedHatAI/Qwen2.5-VL-72B-Instruct-FP8-dynamic.json"
    }
  },
  "kv_offload_support": {
    "offloading_cpu": "verified",
    "offloading_fs": "verified"
  },
  "hardware_overrides": {
    "hopper": {
      "extra_args": [
        "--mm-encoder-tp-mode",
        "data"
      ],
      "extra_env": {}
    },
    "blackwell": {
      "extra_args": [
        "--mm-encoder-tp-mode",
        "data"
      ],
      "extra_env": {}
    },
    "amd": {
      "extra_args": [
        "--mm-encoder-tp-mode",
        "data",
        "--limit-mm-per-prompt",
        "{\"image\":2,\"video\":0}"
      ],
      "extra_env": {
        "VLLM_ROCM_USE_AITER": "1"
      }
    }
  },
  "guide": "## Overview\n\nThis guide describes how to run Qwen2.5-VL series on the targeted accelerated stack. Since BF16 is the commonly used precision for Qwen2.5-VL training, using BF16 in inference ensures the best accuracy.\n\n## Prerequisites\n\n### NVIDIA\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto\n```\n\n### AMD ROCm (MI300X, MI325X, MI355X)\n> Note: The vLLM wheel for ROCm requires Python 3.12, ROCm 7.0, and glibc >= 2.35. Use the Docker flow if your environment is incompatible.\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm\n```\n\n### TPU Deployment\nTPU support has been verified for the smaller [Qwen2.5-VL-7B-Instruct](/Qwen/Qwen2.5-VL-7B-Instruct) on a single v6e chip — the 72B is not yet validated on Trillium. See the [upstream TPU recipe](https://github.com/AI-Hypercomputer/tpu-recipes/tree/main/inference/trillium/vLLM/Qwen2.5-VL) (7B).\n\n## Deployment Configurations\n\n### 4xA100 (BF16, TP=4)\n```bash\nexport CUDA_VISIBLE_DEVICES=0,1,2,3\nvllm serve Qwen/Qwen2.5-VL-72B-Instruct  \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --tensor-parallel-size 4 \\\n  --mm-encoder-tp-mode data \\\n  --limit-mm-per-prompt '{\"image\":2,\"video\":0}'\n```\n\n### 4xMI300X/MI325X/MI355X (BF16, TP=4)\n```bash\nexport CUDA_VISIBLE_DEVICES=0,1,2,3\nexport VLLM_ROCM_USE_AITER=1\nvllm serve Qwen/Qwen2.5-VL-72B-Instruct  \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --tensor-parallel-size 4 \\\n  --mm-encoder-tp-mode data \\\n  --limit-mm-per-prompt '{\"image\":2,\"video\":0}'\n```\n\n### Qwen2.5-VL-7B-Instruct (DP=4)\nFor medium-size 7B model, data parallelism works better.\n\n```bash\nexport CUDA_VISIBLE_DEVICES=0,1,2,3\nvllm serve Qwen/Qwen2.5-VL-7B-Instruct  \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --data-parallel-size 4 \\\n  --limit-mm-per-prompt '{\"image\":2,\"video\":0}'\n```\n\n## Configuration Tips\n\n- `--max-model-len 65536` is usually good for most scenarios (native context is 128K).\n- For A100-80GB devices, TP must be >= 2 to avoid OOM.\n- `--limit-mm-per-prompt` caps incoming multimodal requests.\n- `--mm-encoder-tp-mode data` deploys the small ViT encoder in DP fashion (ViT ≈ 675M vs 72B LM).\n- vLLM uses 90% of GPU memory by default; set `--gpu-memory-utilization=0.95` to maximize KV cache.\n\n## Benchmarking\n\nLaunch the server with `--no-enable-prefix-caching` to get consistent measurements.\n\n### VisionArena-Chat\n```bash\nvllm bench serve \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --backend openai-chat \\\n  --endpoint /v1/chat/completions \\\n  --model Qwen/Qwen2.5-VL-72B-Instruct \\\n  --dataset-name hf \\\n  --dataset-path lmarena-ai/VisionArena-Chat \\\n  --num-prompts 128\n```\n\n### Random Synthetic\n```bash\nvllm bench serve \\\n  --host 0.0.0.0 \\\n  --port 8000 \\\n  --model Qwen/Qwen2.5-VL-72B-Instruct \\\n  --dataset-name random \\\n  --random-input-len 8000 \\\n  --random-output-len 1000 \\\n  --num-prompts 128\n```\n\nWorkload mixes:\n- Prompt-heavy: 8000 in / 1000 out\n- Decode-heavy: 1000 in / 8000 out\n- Balanced: 1000 in / 1000 out\n\n## References\n\n- [Model card](https://huggingface.co/Qwen/Qwen2.5-VL-72B-Instruct)\n- [vLLM multimodal inputs guide](https://docs.vllm.ai/en/latest/features/multimodal_inputs.html)\n",
  "source": {
    "source": "vllm",
    "repo": "vllm-project/vllm",
    "ref": "main",
    "sha": "389ffebe555705c85337025d5df108d0384d365e",
    "path": "recipes/models/Qwen/Qwen2.5-VL-72B-Instruct.yaml",
    "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/Qwen/Qwen2.5-VL-72B-Instruct.yaml",
    "overlays": []
  }
}