{ "hf_id": "tencent/Hy3-preview", "meta": { "title": "Hy3-preview", "slug": "hy3-preview", "provider": "Hunyuan (Tencent)", "description": "Tencent Hunyuan Hy3-preview — scaled-up MoE language model (295B total / 21B active) with a 3.8B MTP layer for speculative decoding, 256K context, and hy_v3 tool/reasoning parsers", "date_added": "2026-04-23", "date_updated": "2026-09-14", "tasks": [ "text" ], "performance_headline": "Hunyuan Hy3-preview MoE — 295B/21B on 8×H200, 8×H20-3e(141GB), or 8×AMD MI300X/MI355X with MTP", "related_recipes": [], "hardware": { "h200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": { "VLLM_USE_RUST_FRONTEND": "1" }, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve tencent/Hy3-preview \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --reasoning-parser hy_v3", "argv": [ "vllm", "serve", "tencent/Hy3-preview", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "hy_v3", "--enable-auto-tool-choice", "--reasoning-parser", "hy_v3" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e VLLM_USE_RUST_FRONTEND=1 \\\n vllm/vllm-openai:latest tencent/Hy3-preview \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --reasoning-parser hy_v3", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "VLLM_USE_RUST_FRONTEND=1", "vllm/vllm-openai:latest", "tencent/Hy3-preview", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "hy_v3", "--enable-auto-tool-choice", "--reasoning-parser", "hy_v3" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/tencent/Hy3-preview/strategies/single_node_tep.json", "single_node_dep": "/tencent/Hy3-preview/strategies/single_node_dep.json", "multi_node_tp": "/tencent/Hy3-preview/strategies/multi_node_tp.json", "multi_node_tep": "/tencent/Hy3-preview/strategies/multi_node_tep.json", "multi_node_dep": "/tencent/Hy3-preview/strategies/multi_node_dep.json", "kv_store_centralized_mooncake": "/tencent/Hy3-preview/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/tencent/Hy3-preview/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/tencent/Hy3-preview/hw/h200.json", "h100": "/tencent/Hy3-preview/hw/h100.json", "b200": "/tencent/Hy3-preview/hw/b200.json", "gb200": "/tencent/Hy3-preview/hw/gb200.json", "b300": "/tencent/Hy3-preview/hw/b300.json", "gb300": "/tencent/Hy3-preview/hw/gb300.json", "mi300x": "/tencent/Hy3-preview/hw/mi300x.json", "mi325x": "/tencent/Hy3-preview/hw/mi325x.json", "mi350x": "/tencent/Hy3-preview/hw/mi350x.json", "mi355x": "/tencent/Hy3-preview/hw/mi355x.json" } }, "model": { "model_id": "tencent/Hy3-preview", "min_vllm_version": "0.20.0", "default_frontend": "rust", "architecture": "moe", "parameter_count": "295B", "active_parameters": "21B", "context_length": 262144, "base_args": [ "--trust-remote-code" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest", "note": "Use the dedicated hy3-preview image until changes land in vllm:latest." } } }, "features": { "tool_calling": { "description": "Hunyuan v3 tool call parser with automatic tool choice", "args": [ "--tool-call-parser", "hy_v3", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "Hunyuan v3 reasoning parser for thinking-mode chain-of-thought extraction", "args": [ "--reasoning-parser", "hy_v3" ] }, "spec_decoding": { "description": "Multi-Token Prediction speculative decoding using the model's built-in MTP layer", "args": [ "--speculative-config", "{\"method\":\"mtp\",\"num_speculative_tokens\":1}" ] } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 708, "description": "Full precision BF16 — 8×H200 or 8×H20-3e(141GB) minimum for weights + KV cache" } }, "hardware_overrides": { "amd": { "install_note": "Hy3-preview model code is being added in PR #40681. Until it merges, build\nvLLM editable from the PR branch in rocm/vllm-dev:nightly:\n\n docker run -it --device=/dev/kfd --device=/dev/dri --network=host \\\n --ipc=host --shm-size=128g --group-add video --cap-add SYS_PTRACE \\\n --security-opt seccomp=unconfined -v ~/work:/work -w /work \\\n -e PYTHONPATH=/work/vllm rocm/vllm-dev:nightly bash\n git clone -b feature/support_hy_v3 \\\n https://github.com/stevenkuang-tencent/vllm.git\n cd vllm && pip uninstall -y vllm\n SETUPTOOLS_SCM_PRETEND_VERSION=0.20.0.dev0 VLLM_TARGET_DEVICE=rocm \\\n pip install --editable . --no-build-isolation\n\nSetting PYTHONPATH avoids a known editable-install conflict with the\nempty /app/vllm namespace directory shipped in the base image.\n", "extra_args": [], "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_MHA": "1", "VLLM_ROCM_USE_AITER_RMSNORM": "1", "VLLM_ROCM_USE_AITER_LINEAR": "1" } } }, "guide": "# Hy3-preview Usage Guide\n\nHy3-preview is Tencent Hunyuan's latest open-source Mixture-of-Experts language model:\n295B total parameters with 21B activated per token, plus a 3.8B MTP layer for\nspeculative decoding. 80 transformer layers, 192 routed experts (top-8) + 1 shared\nexpert, GQA with 64 heads over 8 KV heads, 256K context.\n\nA pretrained base checkpoint is published at\n[tencent/Hy3-preview-Base](https://huggingface.co/tencent/Hy3-preview-Base); this\nrecipe covers the instruct model.\n\n## Setup\n\nChoose one of the following setup methods.\n\n### Using Docker\n\n```bash\ndocker run --gpus all \\\n -p 8000:8000 \\\n --ipc=host \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:hy3-preview tencent/Hy3-preview \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3-preview\n```\n\n### Installing from source\n\n```bash\nuv venv --python 3.12 --seed --managed-python\nsource .venv/bin/activate\ngit clone https://github.com/vllm-project/vllm.git\ncd vllm\nuv pip install --editable . --torch-backend=auto\n```\n\n## Model Deployment\n\nTo serve Hy3-preview on 8 GPUs, use H20-3e(141GB), H200, AMD MI300X/MI325X (192 GB),\nAMD MI350X/MI355X (288 GB), or other GPUs with larger memory capacity. Smaller-memory\n8-GPU configurations (8×H100 80GB, 8×A100 80GB) do not fit the BF16 weights plus KV\ncache — use multi-node TP for those.\n\n### Serving on 8×AMD MI300X / MI325X / MI350X / MI355X\n\nHy3-preview support is being added in vLLM PR\n[#40681](https://github.com/vllm-project/vllm/pull/40681). Until it merges, AMD users\nmust build vLLM from the PR branch inside the published ROCm vLLM nightly image\n(`rocm/vllm-dev:nightly`). See the AMD install note in `hardware_overrides.amd`\nabove for the full reproducer.\n\nOnce vLLM is installed, serve with the standard launcher plus the AITER environment\nvariables (the recipe's `hardware_overrides.amd.extra_env` applies these\nautomatically when the AMD profile is selected on the recipe site):\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_USE_AITER_MOE=1\nexport VLLM_ROCM_USE_AITER_MHA=1\nexport VLLM_ROCM_USE_AITER_RMSNORM=1\nexport VLLM_ROCM_USE_AITER_LINEAR=1\n\nvllm serve tencent/Hy3-preview \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3-preview \\\n --gpu-memory-utilization 0.90\n```\n\nMTP (recommended on AMD for lower latency, same flags as the NVIDIA path):\n\n```bash\nvllm serve tencent/Hy3-preview \\\n --tensor-parallel-size 8 \\\n --speculative-config.method mtp \\\n --speculative-config.num_speculative_tokens 1 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3-preview \\\n --gpu-memory-utilization 0.90\n```\n\n### Serving on 8×H200 or 8×H20-3e(141GB)\n\nWithout Multi-Token Prediction (MTP):\n\n```bash\nvllm serve tencent/Hy3-preview \\\n --tensor-parallel-size 8 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3-preview\n```\n\nWith MTP (recommended for lower latency):\n\n```bash\nvllm serve tencent/Hy3-preview \\\n --tensor-parallel-size 8 \\\n --speculative-config.method mtp \\\n --speculative-config.num_speculative_tokens 1 \\\n --tool-call-parser hy_v3 \\\n --reasoning-parser hy_v3 \\\n --enable-auto-tool-choice \\\n --served-model-name hy3-preview\n```\n\n## Sampling and Reasoning Modes\n\nTencent's recommended sampling parameters: `temperature=0.9`, `top_p=1.0`.\n\nReasoning is controlled via `chat_template_kwargs.reasoning_effort`:\n\n| Value | Behavior |\n|:---|:---|\n| `no_think` (default) | Direct response, no chain-of-thought |\n| `low` | Light reasoning |\n| `high` | Deep chain-of-thought for math/coding/complex reasoning |\n\nWhen tools are registered, set `interleaved_thinking: true` to allow the model to\nthink between tool calls.\n\n### OpenAI Client Example\n\n```bash\nuv pip install -U openai\n```\n\n```python\nfrom openai import OpenAI\n\nclient = OpenAI(base_url=\"http://localhost:8000/v1\", api_key=\"EMPTY\")\nmessages = [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": \"Hello.\"},\n]\n\n# Direct response (default).\nresp = client.chat.completions.create(\n model=\"hy3-preview\",\n messages=messages,\n temperature=0.9,\n top_p=1.0,\n max_tokens=4096,\n)\nprint(resp.choices[0].message.content)\n\n# Deep reasoning: set reasoning_effort (and interleaved_thinking if using tools).\nresp_think = client.chat.completions.create(\n model=\"hy3-preview\",\n messages=messages,\n temperature=0.9,\n top_p=1.0,\n max_tokens=4096,\n extra_body={\n \"chat_template_kwargs\": {\n \"reasoning_effort\": \"high\",\n \"interleaved_thinking\": True,\n },\n },\n)\noutput_msg = resp_think.choices[0].message\nprint(output_msg.reasoning) # chain-of-thought\nprint(output_msg.content) # final answer\n```\n\n### cURL Usage\n\n```bash\ncurl http://localhost:8000/v1/chat/completions \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"model\": \"hy3-preview\",\n \"messages\": [\n {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n {\"role\": \"user\", \"content\": \"Hello.\"}\n ],\n \"temperature\": 0.9,\n \"top_p\": 1.0,\n \"max_tokens\": 4096\n }'\n```\n\n## Benchmarking\n\nFor benchmarking, disable prefix caching by adding `--no-enable-prefix-caching` to\nthe server command.\n\nThe following uses 8×H20-3e(141GB) as an example.\n\n```bash\nvllm bench serve \\\n --model tencent/Hy3-preview \\\n --dataset-name random \\\n --random-input-len 8192 \\\n --random-output-len 1024 \\\n --max-concurrency 32 \\\n --num-prompts 160 \\\n --served-model-name hy3-preview\n```\n\nRepresentative output:\n\n```shell\n============ Serving Benchmark Result ============\nSuccessful requests: 160 \nFailed requests: 0 \nMaximum request concurrency: 32 \nBenchmark duration (s): 280.58 \nTotal input tokens: 1310720 \nTotal generated tokens: 163840 \nRequest throughput (req/s): 0.57 \nOutput token throughput (tok/s): 583.93 \nPeak output token throughput (tok/s): 1024.00 \nPeak concurrent requests: 36.00 \nTotal token throughput (tok/s): 5255.36 \n---------------Time to First Token----------------\nMean TTFT (ms): 4542.41 \nMedian TTFT (ms): 2762.17 \nP99 TTFT (ms): 21062.96 \n-----Time per Output Token (excl. 1st token)------\nMean TPOT (ms): 50.34 \nMedian TPOT (ms): 51.77 \nP99 TPOT (ms): 54.07 \n---------------Inter-token Latency----------------\nMean ITL (ms): 50.34 \nMedian ITL (ms): 34.32 \nP99 ITL (ms): 689.10 \n==================================================\n```\n## References\n\n- [Hugging Face model card](https://huggingface.co/tencent/Hy3-preview)\n- [Hy3-preview-Base (pretrained)](https://huggingface.co/tencent/Hy3-preview-Base)\n- [GitHub: Tencent-Hunyuan/Hy3-preview](https://github.com/Tencent-Hunyuan/Hy3-preview)\n- [Tencent Hunyuan](https://hy.tencent.com/research/hy3)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/tencent/Hy3-preview.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/tencent/Hy3-preview.yaml", "overlays": [] } }