{ "hf_id": "deepseek-ai/DeepSeek-V3", "meta": { "title": "DeepSeek-V3", "slug": "deepseek-v3", "provider": "DeepSeek", "description": "DeepSeek-V3 is a 671B-parameter Mixture-of-Experts model with native FP8 weights and strong reasoning, coding, and math capabilities.", "date_added": "2025-08-01", "date_updated": "2026-09-10", "tasks": [ "text" ], "performance_headline": "Frontier open-weights MoE with native FP8 and FP4 variants", "related_recipes": [ "deepseek-ai/DeepSeek-R1", "deepseek-ai/DeepSeek-V3.1" ], "hardware": { "h200": "verified", "b200": "verified", "gb200": "verified", "mi300x": "verified", "mi325x": "verified", "mi350x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve deepseek-ai/DeepSeek-V3 \\\n --trust-remote-code \\\n --enable-expert-parallel \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser deepseek_v3 \\\n --chat-template examples/tool_chat_template_deepseekv3.jinja \\\n --reasoning-parser deepseek_v3", "argv": [ "vllm", "serve", "deepseek-ai/DeepSeek-V3", "--trust-remote-code", "--enable-expert-parallel", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "deepseek_v3", "--chat-template", "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser", "deepseek_v3" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest deepseek-ai/DeepSeek-V3 \\\n --trust-remote-code \\\n --enable-expert-parallel \\\n --tensor-parallel-size 8 \\\n --enable-auto-tool-choice \\\n --tool-call-parser deepseek_v3 \\\n --chat-template examples/tool_chat_template_deepseekv3.jinja \\\n --reasoning-parser deepseek_v3", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "deepseek-ai/DeepSeek-V3", "--trust-remote-code", "--enable-expert-parallel", "--tensor-parallel-size", "8", "--enable-auto-tool-choice", "--tool-call-parser", "deepseek_v3", "--chat-template", "examples/tool_chat_template_deepseekv3.jinja", "--reasoning-parser", "deepseek_v3" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/deepseek-ai/DeepSeek-V3/strategies/single_node_tep.json", "single_node_dep": "/deepseek-ai/DeepSeek-V3/strategies/single_node_dep.json", "single_node_dpa_tp": "/deepseek-ai/DeepSeek-V3/strategies/single_node_dpa_tp.json", "multi_node_tp": "/deepseek-ai/DeepSeek-V3/strategies/multi_node_tp.json", "multi_node_tp_pp": "/deepseek-ai/DeepSeek-V3/strategies/multi_node_tp_pp.json", "multi_node_dep": "/deepseek-ai/DeepSeek-V3/strategies/multi_node_dep.json", "multi_node_tep": "/deepseek-ai/DeepSeek-V3/strategies/multi_node_tep.json", "pd_cluster": "/deepseek-ai/DeepSeek-V3/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/deepseek-ai/DeepSeek-V3/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/deepseek-ai/DeepSeek-V3/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/deepseek-ai/DeepSeek-V3/hw/h200.json", "h100": "/deepseek-ai/DeepSeek-V3/hw/h100.json", "b200": "/deepseek-ai/DeepSeek-V3/hw/b200.json", "gb200": "/deepseek-ai/DeepSeek-V3/hw/gb200.json", "b300": "/deepseek-ai/DeepSeek-V3/hw/b300.json", "gb300": "/deepseek-ai/DeepSeek-V3/hw/gb300.json", "mi300x": "/deepseek-ai/DeepSeek-V3/hw/mi300x.json", "mi325x": "/deepseek-ai/DeepSeek-V3/hw/mi325x.json", "mi350x": "/deepseek-ai/DeepSeek-V3/hw/mi350x.json", "mi355x": "/deepseek-ai/DeepSeek-V3/hw/mi355x.json" } }, "model": { "model_id": "deepseek-ai/DeepSeek-V3", "min_vllm_version": "0.12.0", "architecture": "moe", "parameter_count": "671B", "active_parameters": "37B", "context_length": 163840, "supports_dcp": true, "base_args": [ "--trust-remote-code", "--enable-expert-parallel" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Enable DeepSeek-V3 tool calling with the deepseek_v3 tool-call parser.", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "deepseek_v3", "--chat-template", "examples/tool_chat_template_deepseekv3.jinja" ] }, "reasoning": { "description": "Enable reasoning/thinking mode with the DeepSeek R1 reasoning parser.", "args": [ "--reasoning-parser", "deepseek_v3" ] } }, "opt_in_features": [], "variants": { "default": { "precision": "fp8", "vram_minimum_gb": 805, "description": "Native FP8 weights on 8xH200 (recommended)" }, "v3_0324": { "model_id": "deepseek-ai/DeepSeek-V3-0324", "precision": "fp8", "vram_minimum_gb": 805, "description": "March 2025 DeepSeek-V3 refresh (DeepSeek-V3-0324)", "json": "/deepseek-ai/DeepSeek-V3-0324.json" }, "fp4": { "model_id": "nvidia/DeepSeek-V3-FP4", "precision": "fp4", "vram_minimum_gb": 403, "description": "NVIDIA FP4 quantized weights for Blackwell (e.g. 4xB200)", "json": "/nvidia/DeepSeek-V3-FP4.json" } }, "hardware_overrides": { "amd": { "extra_args": [], "extra_env": { "VLLM_ROCM_USE_AITER": "1" } } }, "guide": "## Overview\n\nDeepSeek-V3 is a 671B-parameter Mixture-of-Experts model (37B activated per token)\nshipped with native FP8 weights. It shares its architecture with DeepSeek-R1, so the\nsame launch recipes apply to both models. DeepSeek publishes a refreshed checkpoint as\n[`DeepSeek-V3-0324`](https://huggingface.co/deepseek-ai/DeepSeek-V3-0324), available\nhere as the **V3 0324** variant. For Blackwell GPUs, NVIDIA publishes an FP4 quantized\nvariant (`nvidia/DeepSeek-V3-FP4` / `nvidia/DeepSeek-R1-FP4`) that runs on fewer GPUs.\n\n## Prerequisites\n\n- **Hardware (FP8)**: 8x H200 (CUDA) or 8x MI300X / MI325X / MI355X (ROCm)\n- **Hardware (FP4)**: 4x B200 GPUs\n- **vLLM**: Current stable release\n\n### CUDA\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto\n```\n\n### ROCm (MI300X, MI325X, MI355X)\nRequires Python 3.12, ROCm 7.2.1, and glibc >= 2.35.\n```bash\nuv venv --python 3.12\nsource .venv/bin/activate\nuv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/\n```\n\n## Client Usage\n\n### 8xH200 / 8xMI300X (FP8)\n\nTensor Parallel + Expert Parallel (TP8+EP) — CUDA:\n```bash\nvllm serve deepseek-ai/DeepSeek-V3 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel\n```\n\nTensor Parallel + Expert Parallel (TP8+EP) — ROCm:\n```bash\nexport VLLM_ROCM_USE_AITER=1\n\nvllm serve deepseek-ai/DeepSeek-V3 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --enable-expert-parallel\n```\n\nData Parallel + Expert Parallel (DP8+EP) — CUDA:\n```bash\nvllm serve deepseek-ai/DeepSeek-V3 \\\n --trust-remote-code \\\n --data-parallel-size 8 \\\n --enable-expert-parallel\n```\n\n### 8xMI350X / 8xMI355X (DPA+TP)\n\nEach GPU runs its own attention replica with a private KV cache, while the MoE\nexperts stay tensor-sharded across all eight GPUs. Aggregate KV-cache\ncapacity is roughly 8x that of a single TP8 replica, useful for workloads \nbenefiting from larger KV capacities. \n\nEngine (all eight ranks in one process, listening on 8100):\n```bash\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ENGINE_READY_TIMEOUT_S=1800\n\nvllm serve deepseek-ai/DeepSeek-V3-0324 \\\n --trust-remote-code \\\n --port 8100 \\\n --data-parallel-size 8 \\\n --tensor-parallel-size 1\n```\n\nRouter (client-facing on 8000, spreads requests across the eight DP ranks):\n```bash\nuv pip install vllm-router\n\nvllm-router \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --policy consistent_hash \\\n --intra-node-data-parallel-size 8 \\\n --worker-urls http://localhost:8100 \\\n --worker-startup-timeout-secs 1800\n```\n\nNotes:\n\n- `--intra-node-data-parallel-size` must match the engine's\n `--data-parallel-size`. A mismatch silently under-uses ranks.\n- `consistent_hash` is the policy that makes this worthwhile for chat: it pins\n a conversation to the rank that already holds its prefix, so the larger\n aggregate cache actually gets hit. `round_robin` scatters turns of the same\n conversation across ranks and throws the prefix away.\n- Start the engine first, then the router; `--worker-startup-timeout-secs 1800`\n covers the engine's weight load either way.\n- Send client traffic to the router on port 8000, never to 8100 directly.\n\n### 4xB200 (FP4)\n\nOn vLLM v0.28.0 and later, the FlashInfer MoE kernels are selected automatically\nfor both FP4 and FP8 on Blackwell — no environment variables are needed. To pin\nthe backend explicitly, pass `--moe-backend flashinfer_trtllm`.\n\nTensor Parallel + Expert Parallel (TP4+EP):\n```bash\nCUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve nvidia/DeepSeek-V3-FP4 \\\n --trust-remote-code \\\n --tensor-parallel-size 4 \\\n --enable-expert-parallel\n```\n\nData Parallel + Expert Parallel (DP4+EP):\n```bash\nCUDA_VISIBLE_DEVICES=0,1,2,3 vllm serve nvidia/DeepSeek-V3-FP4 \\\n --trust-remote-code \\\n --data-parallel-size 4 \\\n --enable-expert-parallel\n```\n\n## Benchmarking\n\nFor benchmarking, prefix caching is disabled by default in vLLM — no extra server\nflag is needed.\n\n```bash\n# Prompt-heavy benchmark (8k/1k)\nvllm bench serve \\\n --model deepseek-ai/DeepSeek-V3 \\\n --dataset-name random \\\n --random-input-len 8000 \\\n --random-output-len 1000 \\\n --request-rate 10000 \\\n --num-prompts 16 \\\n --ignore-eos\n```\n\nTest different workloads by adjusting input/output lengths:\n- Prompt-heavy: 8000 input / 1000 output\n- Decode-heavy: 1000 input / 8000 output\n- Balanced: 1000 input / 1000 output\n\n## Troubleshooting\n\n- **Disaggregated Serving with Wide EP (Experimental GB200)**: See\n [vLLM issue #33583](https://github.com/vllm-project/vllm/issues/33583) and the\n [vLLM blog post](https://blog.vllm.ai/2026/02/03/dsr1-gb200-part1.html) for\n GB200 disaggregated serving recipes.\n\n## References\n\n- [DeepSeek-V3 on Hugging Face](https://huggingface.co/deepseek-ai/DeepSeek-V3)\n- [DeepSeek-V3-0324 on Hugging Face](https://huggingface.co/deepseek-ai/DeepSeek-V3-0324)\n- [NVIDIA DeepSeek-V3-FP4](https://huggingface.co/nvidia/DeepSeek-V3-FP4)\n- [vLLM Expert Parallelism docs](https://docs.vllm.ai/en/latest/serving/expert_parallel_deployment.html)\n- [vLLM Data Parallel deployment docs](https://docs.vllm.ai/en/latest/serving/data_parallel_deployment.html)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/deepseek-ai/DeepSeek-V3.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/deepseek-ai/DeepSeek-V3.yaml", "overlays": [] } }