{ "hf_id": "tencent/Hy4-preview", "meta": { "title": "Hy4-preview", "slug": "hy4-preview", "provider": "Hunyuan (Tencent)", "description": "Tencent Hunyuan Hy4-preview — scaled-up MoE language model (770B total / 49B active) with a 10B MTP layer for speculative decoding, 1M context, and hy_v4 tool/reasoning parsers", "date_added": "2026-08-28", "date_updated": "2026-09-28", "tasks": [ "text" ], "performance_headline": "Hunyuan Hy4-preview MoE — 770B/49B on 16xB200, 8xB300, 8xMI355X with MTP", "related_recipes": [ "tencent/Hy3", "tencent/Hy3-preview" ], "default_hardware": "b300", "hardware": { "b200": "verified", "b300": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "b300", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_ENABLE_HPC_OPS": "1", "VLLM_USE_RUST_FRONTEND": "1" }, "docker_image": "vllm/vllm-openai:hy4-preview", "command": "vllm serve tencent/Hy4-preview \\\n --tensor-parallel-size 8 \\\n --attention-backend FLASHMLA_SPARSE \\\n --tool-call-parser hy_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser hy_v4", "argv": [ "vllm", "serve", "tencent/Hy4-preview", "--tensor-parallel-size", "8", "--attention-backend", "FLASHMLA_SPARSE", "--tool-call-parser", "hy_v4", "--enable-auto-tool-choice", "--reasoning-parser", "hy_v4" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_ENABLE_HPC_OPS=1 \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n vllm/vllm-openai:hy4-preview tencent/Hy4-preview \\\n --tensor-parallel-size 8 \\\n --attention-backend FLASHMLA_SPARSE \\\n --tool-call-parser hy_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser hy_v4", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_ENABLE_HPC_OPS=1", "-e", "VLLM_USE_RUST_FRONTEND=1", "vllm/vllm-openai:hy4-preview", "tencent/Hy4-preview", "--tensor-parallel-size", "8", "--attention-backend", "FLASHMLA_SPARSE", "--tool-call-parser", "hy_v4", "--enable-auto-tool-choice", "--reasoning-parser", "hy_v4" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "blackwell", "compute_arch": "sm103", "display_name": "B300", "description": "NVIDIA B300 SXM 268 GB HBM3e · 8-GPU HGX B300 node (Blackwell Ultra)", "gpu_count": 8, "vram_gb": 2144, "multi_node": false }, "alternatives": { "single_node_tep": "/tencent/Hy4-preview/strategies/single_node_tep.json", "single_node_dep": "/tencent/Hy4-preview/strategies/single_node_dep.json", "multi_node_tp": "/tencent/Hy4-preview/strategies/multi_node_tp.json", "multi_node_tep": "/tencent/Hy4-preview/strategies/multi_node_tep.json", "multi_node_dep": "/tencent/Hy4-preview/strategies/multi_node_dep.json", "kv_store_centralized_mooncake": "/tencent/Hy4-preview/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/tencent/Hy4-preview/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "b300": "/tencent/Hy4-preview/hw/b300.json", "h100": "/tencent/Hy4-preview/hw/h100.json", "h200": "/tencent/Hy4-preview/hw/h200.json", "b200": "/tencent/Hy4-preview/hw/b200.json", "gb200": "/tencent/Hy4-preview/hw/gb200.json", "gb300": "/tencent/Hy4-preview/hw/gb300.json", "mi300x": "/tencent/Hy4-preview/hw/mi300x.json", "mi325x": "/tencent/Hy4-preview/hw/mi325x.json", "mi350x": "/tencent/Hy4-preview/hw/mi350x.json", "mi355x": "/tencent/Hy4-preview/hw/mi355x.json" } }, "model": { "model_id": "tencent/Hy4-preview", "min_vllm_version": "0.29.0", "default_frontend": "rust", "docker_image": { "nvidia": "vllm/vllm-openai:hy4-preview", "amd": "vllm/vllm-openai-rocm:nightly" }, "nightly_required": true, "architecture": "moe", "parameter_count": "770B", "active_parameters": "49B", "context_length": 1048576, "base_args": [], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --pre \\\n --extra-index-url https://wheels.vllm.ai/nightly/cu130 \\\n --extra-index-url https://download.pytorch.org/whl/cu130 \\\n --index-strategy unsafe-best-match", "note": "Hy4-preview's latest optimizations aren't in a stable release yet — use nightly wheels or build from source." }, "docker": { "command": "docker pull vllm/vllm-openai:hy4-preview", "note": "NVIDIA: dedicated hy4-preview image until changes land in vllm:latest. AMD: vllm/vllm-openai-rocm:nightly from 2026-09-22 or later." } } }, "features": { "tool_calling": { "description": "Hunyuan v4 tool call parser with automatic tool choice", "args": [ "--tool-call-parser", "hy_v4", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "Hunyuan v4 reasoning parser for thinking-mode chain-of-thought extraction", "args": [ "--reasoning-parser", "hy_v4" ] }, "spec_decoding": { "description": "Multi-Token Prediction speculative decoding using the model's built-in MTP layer", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":3}" ] } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 1848, "description": "Full precision BF16 — 16xB200 or 8xB300 minimum for weights + KV cache", "hardware_overrides": { "amd": { "extra_args": [ "--gpu-memory-utilization", "0.92" ] } } }, "fp8": { "model_id": "tencent/Hy4-preview-FP8", "precision": "mxfp8", "vram_minimum_gb": 924, "description": "MXFP8-quantized instruct checkpoint (tencent/Hy4-preview-FP8) — fits a single 8xB200 / 8xB300 node with wide headroom. Verified AMD path on 8xMI355X and 8xMI325X.", "json": "/tencent/Hy4-preview-FP8.json" } }, "hardware_overrides": { "blackwell": { "extra_env": { "VLLM_ENABLE_HPC_OPS": "1" }, "extra_args": [ "--attention-backend", "FLASHMLA_SPARSE" ] }, "amd": { "extra_env": { "VLLM_ROCM_USE_AITER": "1" }, "extra_args": [ "--max-model-len", "262144", "--gpu-memory-utilization", "0.85", "--max-num-batched-tokens", "8192" ] } }, "guide": "# Hy4-preview Usage Guide\n\nHy4-preview is Tencent Hunyuan's new-generation open-source Mixture-of-Experts language\nmodel. Its backbone has 770B total parameters with 49B activated per token across\n78 layers. The first layer uses a dense FFN; the other 77 layers use 256 routed\nexperts (top-8) and 1 shared expert. A native MTP layer adds 10B total parameters\n(0.7B activated) for speculative decoding.\n\nOn the architecture side, inspired by DeepSeek and GLM, the attention module employs Gated DeepSeek Sparse Attention (Gated DSA) with IndexCache for cross-layer sparse index reuse. The residual pathway uses iHC (identity Hyper-Connections) to expand inter-layer information flow.\n\n## vLLM Setup\n\nChoose one of the following setup methods.\n\n### Using Docker\n\n```bash\ndocker run --gpus all \\\n -p 8000:8000 \\\n --ipc=host \\\n -e VLLM_ENABLE_HPC_OPS=1 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:hy4-preview tencent/Hy4-preview-FP8 \\\n --tensor-parallel-size 8 \\\n --speculative-config '{\"num_speculative_tokens\":3,\"method\":\"mtp\"}' \\\n --attention-backend FLASHMLA_SPARSE \\\n --tool-call-parser hy_v4 \\\n --reasoning-parser hy_v4 \\\n --enable-auto-tool-choice \\\n --port 8000 \\\n --served-model-name hy4-preview\n```\n\nBuild vLLM from source:\n\n```bash\nuv venv --python 3.12 --seed --managed-python\nsource .venv/bin/activate\ngit clone https://github.com/vllm-project/vllm.git\ncd vllm\nuv pip install --editable . --torch-backend=auto\n```\n\n## vLLM Deployment\n\nStart the FP8 model with MTP enabled:\n\n```bash\nexport VLLM_ENABLE_HPC_OPS=1\n\nvllm serve tencent/Hy4-preview-FP8 \\\n --tensor-parallel-size 8 \\\n --speculative-config '{\"num_speculative_tokens\":3,\"method\":\"mtp\"}' \\\n --attention-backend FLASHMLA_SPARSE \\\n --tool-call-parser hy_v4 \\\n --reasoning-parser hy_v4 \\\n --enable-auto-tool-choice \\\n --port 8000 \\\n --served-model-name hy4-preview\n```\n\n### Serving on 8×AMD MI355X (gfx950)\n\nSpec decoding (native MTP, `k=3`) is opt-in. \n\n```bash\nexport VLLM_ROCM_USE_AITER=1\n\nvllm serve tencent/Hy4-preview-FP8 \\\n --tensor-parallel-size 8 \\\n --max-model-len 262144 \\\n --gpu-memory-utilization 0.85 \\\n --max-num-batched-tokens 8192 \\\n --tool-call-parser hy_v4 \\\n --reasoning-parser hy_v4 \\\n --enable-auto-tool-choice \\\n --port 8000 \\\n --served-model-name hy4-preview\n```\n\n**Context window.** 262,144 is the default verified window. MXFP8\n(`tencent/Hy4-preview-FP8`) also comes up at the full 1,048,576 ceiling\nat util `0.92` (1.21× concurrency for a 1M request). \n\n### Serving on 8×AMD MI325X (gfx942)\n\nSelect the FP8 variant. Since gfx942 dequantizes MXFP8 weights to BF16,\nthe context window is not the same as on the MI355X. The verified serve\nis context 139264, FP8 KV, and the AITER MoE, MHA, RMSNorm, and linear toggles.\nAt GPU memory utilization 0.96 the KV cache holds 530,912 tokens,\nabout 3.81× one 139,264-token request.\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_USE_AITER_MOE=1\nexport VLLM_ROCM_USE_AITER_MHA=1\nexport VLLM_ROCM_USE_AITER_RMSNORM=1\nexport VLLM_ROCM_USE_AITER_LINEAR=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\nexport VLLM_USE_RUST_FRONTEND=1\n\nvllm serve tencent/Hy4-preview-FP8 \\\n --tensor-parallel-size 8 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --max-model-len 139264 \\\n --max-num-batched-tokens 16384 \\\n --gpu-memory-utilization 0.96 \\\n --tool-call-parser hy_v4 \\\n --enable-auto-tool-choice \\\n --reasoning-parser hy_v4\n```\n\n## OpenAI Client Example\n\nTencent recommends `temperature=0.9` and `top_p=1.0`. Reasoning defaults to\n`high`, which is suitable for math, coding, and other complex tasks.\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(base_url=\"http://127.0.0.1:8000/v1\", api_key=\"EMPTY\")\n\nresponse = client.chat.completions.create(\n model=\"hy4-preview\",\n messages=[\n {\"role\": \"user\", \"content\": \"Hello! Can you briefly introduce yourself?\"},\n ],\n temperature=0.9,\n top_p=1.0,\n)\noutput_msg = response.choices[0].message\nprint(output_msg.reasoning) # chain-of-thought\nprint(output_msg.content) # final answer\n```\n\nFor a direct response without deep reasoning, pass `reasoning_effort=\"no_think\"`:\n\n```python\nresponse = client.chat.completions.create(\n model=\"hy4-preview\",\n messages=[\n {\"role\": \"user\", \"content\": \"Hello! Can you briefly introduce yourself?\"},\n ],\n temperature=0.9,\n top_p=1.0,\n extra_body={\"chat_template_kwargs\": {\"reasoning_effort\": \"no_think\"}},\n)\nprint(response.choices[0].message.content)\n```\n\n### cURL Usage\n\n```bash\ncurl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"hy4-preview\",\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": \"Hello! Can you briefly introduce yourself?\"}\n ],\n \"temperature\": 0.9,\n \"top_p\": 1.0\n }'\n```\n\n## References\n\n- [Hugging Face: Hy4-preview](https://huggingface.co/tencent/Hy4-preview)\n- [Hugging Face: Hy4-preview-FP8](https://huggingface.co/tencent/Hy4-preview-FP8)\n- [ModelScope: Hy4-preview](https://modelscope.cn/models/Tencent-Hunyuan/Hy4-preview)\n- [GitHub: Tencent-Hunyuan/Hy4-preview](https://github.com/Tencent-Hunyuan/Hy4-preview)\n- [GitCode: Hy4-preview](https://ai.gitcode.com/tencent_hunyuan/Hy4-preview)\n- [CNB: Hy4-preview](https://cnb.cool/ai-models/tencent/Hy4-preview)\n- [AngelSlim quantization toolkit](https://github.com/tencent/AngelSlim)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/tencent/Hy4-preview.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/tencent/Hy4-preview.yaml", "overlays": [] } }