{ "hf_id": "moonshotai/Kimi-K2.5", "meta": { "title": "Kimi-K2.5", "slug": "kimi-k2.5", "provider": "Moonshot AI", "description": "Open-source native multimodal agentic MoE model with vision-language understanding, tool calling, and thinking modes", "date_added": "2026-01-27", "date_updated": "2026-07-29", "tasks": [ "multimodal", "text" ], "performance_headline": "Multimodal agentic MoE model with DeepSeek-V3 backbone and MLA attention", "related_recipes": [], "hardware": { "h200": "verified", "gb200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve moonshotai/Kimi-K2.5 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser kimi_k2 \\\n --enable-auto-tool-choice \\\n --reasoning-parser kimi_k2 \\\n --speculative-config '{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}' \\\n --mm-encoder-tp-mode data", "argv": [ "vllm", "serve", "moonshotai/Kimi-K2.5", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "kimi_k2", "--enable-auto-tool-choice", "--reasoning-parser", "kimi_k2", "--speculative-config", "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--mm-encoder-tp-mode", "data" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest moonshotai/Kimi-K2.5 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser kimi_k2 \\\n --enable-auto-tool-choice \\\n --reasoning-parser kimi_k2 \\\n --speculative-config '{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}' \\\n --mm-encoder-tp-mode data", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "moonshotai/Kimi-K2.5", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "kimi_k2", "--enable-auto-tool-choice", "--reasoning-parser", "kimi_k2", "--speculative-config", "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--mm-encoder-tp-mode", "data" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/moonshotai/Kimi-K2.5/strategies/single_node_tep.json", "single_node_dep": "/moonshotai/Kimi-K2.5/strategies/single_node_dep.json", "multi_node_tp": "/moonshotai/Kimi-K2.5/strategies/multi_node_tp.json", "multi_node_tp_pp": "/moonshotai/Kimi-K2.5/strategies/multi_node_tp_pp.json", "multi_node_dep": "/moonshotai/Kimi-K2.5/strategies/multi_node_dep.json", "multi_node_tep": "/moonshotai/Kimi-K2.5/strategies/multi_node_tep.json", "pd_cluster": "/moonshotai/Kimi-K2.5/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/moonshotai/Kimi-K2.5/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/moonshotai/Kimi-K2.5/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/moonshotai/Kimi-K2.5/hw/h200.json", "h100": "/moonshotai/Kimi-K2.5/hw/h100.json", "b200": "/moonshotai/Kimi-K2.5/hw/b200.json", "gb200": "/moonshotai/Kimi-K2.5/hw/gb200.json", "b300": "/moonshotai/Kimi-K2.5/hw/b300.json", "gb300": "/moonshotai/Kimi-K2.5/hw/gb300.json", "mi300x": "/moonshotai/Kimi-K2.5/hw/mi300x.json", "mi325x": "/moonshotai/Kimi-K2.5/hw/mi325x.json", "mi350x": "/moonshotai/Kimi-K2.5/hw/mi350x.json", "mi355x": "/moonshotai/Kimi-K2.5/hw/mi355x.json" } }, "model": { "model_id": "moonshotai/Kimi-K2.5", "min_vllm_version": "0.19.1", "docker_image": { "nvidia": "vllm/vllm-openai:latest", "amd": "vllm/vllm-openai-rocm:nightly" }, "architecture": "moe", "parameter_count": "1T", "active_parameters": "32B", "context_length": 262144, "supports_dcp": true, "base_args": [ "--trust-remote-code" ], "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Kimi K2 tool-call parser with automatic tool choice", "args": [ "--tool-call-parser", "kimi_k2", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "Kimi K2 reasoning parser for extracting chain-of-thought content", "args": [ "--reasoning-parser", "kimi_k2" ] }, "spec_decoding": { "description": "Eagle3 speculative decoding for accelerated inference (requires vLLM >= 0.18.0)", "args": [ "--speculative-config", "{\"model\":\"lightseekorg/kimi-k2.5-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" ] }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "encoder_parallel": { "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.", "args": [ "--mm-encoder-tp-mode", "data" ] } }, "opt_in_features": [ "text_only" ], "hardware_opt_in_features": { "gb200": [ "encoder_parallel" ] }, "variants": { "default": { "precision": "int4", "vram_minimum_gb": 714, "description": "Packed INT4 via compressed-tensors (~595 GB on disk); fits 8×H200" }, "nvfp4": { "model_id": "nvidia/Kimi-K2.5-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 600, "description": "NVIDIA NVFP4 quantized weights for Blackwell GPUs (e.g. GB200)", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/nvidia/Kimi-K2.5-NVFP4.json" }, "mxfp4": { "model_id": "amd/Kimi-K2.5-MXFP4", "precision": "mxfp4", "vram_minimum_gb": 669, "description": "AMD Quark OCP MXFP4 weights for MI355X (gfx950); ~557 GB on disk.", "supported_hardware": [ "mi355x" ], "tp": 8, "extra_args": [ "--kv-cache-dtype", "fp8", "--mm-encoder-tp-mode", "data" ], "extra_env": { "VLLM_ROCM_USE_AITER_MOE": "1", "VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "1", "VLLM_ROCM_USE_SKINNY_GEMM": "0", "AITER_MXFP4_INTERMEDIATE": "1" }, "hardware_overrides": { "mi355x": { "docker_image": "vllm/vllm-openai-rocm:nightly" } }, "json": "/amd/Kimi-K2.5-MXFP4.json" } }, "hardware_overrides": { "amd": { "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "guide": "## Overview\n\nKimi K2.5 is an open-source, native multimodal agentic model built through continual\npretraining on approximately 15 trillion mixed visual and text tokens atop Kimi-K2-Base.\nIt seamlessly integrates vision and language understanding with advanced agentic\ncapabilities, instant and thinking modes, as well as conversational and agentic paradigms.\n\n## Prerequisites\n\n- **vLLM version:** >= 0.15.0 (speculative decoding with Eagle3 requires >= 0.18.0)\n- **Hardware (BF16):** 8x H200 GPUs (verified), or equivalent aggregate VRAM (~640 GB)\n- **Hardware (NVFP4):** 4x Blackwell GPUs (e.g. GB200)\n- **AMD support:** 8x MI300X / MI325X / MI355X with ROCm 7.2.1 and Python 3.12\n\n### Install vLLM\n\n**Pip (NVIDIA):**\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install vllm --torch-backend auto\n```\n\n**Pip (AMD ROCm):**\n```bash\nuv venv --python 3.12\nsource .venv/bin/activate\nuv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm\n```\n\n**Docker (NVIDIA):**\n```bash\ndocker pull vllm/vllm-openai:latest\n```\n\n### AMD MI300X/MI325X\n\nOn 8x MI300X or MI325X (`gfx942`), use the standard W4A16 MoE path with AITER\nand INT4 QuickReduce.\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\n\nvllm serve moonshotai/Kimi-K2.5 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser kimi_k2 \\\n --enable-auto-tool-choice \\\n --reasoning-parser kimi_k2 \\\n --mm-encoder-tp-mode data\n```\n\n### AMD MI350X/MI355X\n\nOn 8x MI350X or MI355X (`gfx950`), add `--moe-backend flydsl` to use the\noptimized FlyDSL W4A16 MoE kernel. Keep LoRA disabled for this path.\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\n\nvllm serve moonshotai/Kimi-K2.5 \\\n --tensor-parallel-size 8 \\\n --trust-remote-code \\\n --mm-encoder-tp-mode data \\\n --moe-backend flydsl \\\n --compilation-config '{\"pass_config\": {\"fuse_allreduce_rms\": false}}'\n```\n\nNotes:\n- The FlyDSL INT4 MoE path does not support expert parallelism; do not add\n `--enable-expert-parallel`.\n- Keep `--compilation-config '{\"pass_config\": {\"fuse_allreduce_rms\": false}}'`;\n it is required for this FlyDSL path on MI350X / MI355X.\n- vLLM has tuned MI350X/MI355X FlyDSL configs for this Kimi shape at TP=8 and TP=4.\n- Keep vLLM's default block size unless you are tuning long-context\n throughput; `--block-size 64` is safe to try.\n\n### AMD MI355X (MXFP4 checkpoint)\n\nThe **MXFP4** variant serves the AMD Quark checkpoint\n[`amd/Kimi-K2.5-MXFP4`](https://huggingface.co/amd/Kimi-K2.5-MXFP4) — a\ndifferent quantization from the packed-INT4 `moonshotai/Kimi-K2.5` FlyDSL path\nabove. It runs the AITER MXFP4 MoE path and adds an fp8 KV cache.\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\n\nvllm serve amd/Kimi-K2.5-MXFP4 \\\n --tensor-parallel-size 8 \\\n --trust-remote-code \\\n --mm-encoder-tp-mode data \\\n --kv-cache-dtype fp8 \\\n --gpu-memory-utilization 0.90\n```\n\nNotes:\n- **fp8 KV cache** is the main MI355X win: it doubles resident KV capacity and\n halves KV read bandwidth. Measured +27% throughput at 8k/1k input with no\n accuracy change (GSM8K 0.9697, unchanged from bf16 KV).\n- **AITER RMSNorm** stays on. A local TP4 GSM8K check on\n the current nightly measured 0.9697.\n- **Old firmware only:** if `rocm-smi --showfw` reports a MEC firmware version\n below 177, add `export HSA_NO_SCRATCH_RECLAIM=1` — older firmware can't\n reclaim RCCL scratch and vLLM crashes without it. Current MI355X firmware\n doesn't need it.\n- For maximum throughput on fixed-length benchmark 8k/1k or 1k/1k workloads,\n see the benchmark reproduction section below. These are throughput-sweep\n tunings; leave vLLM's defaults for general serving.\n\n### MXFP4 benchmark reproduction (InferenceX MI355X sweep)\n\nThe command above is the default serving configuration. The\n[SemiAnalysis InferenceX](https://inferencex.semianalysis.com/) MI355X\nbenchmark lane for this checkpoint is a separate **high-concurrency benchmark**\nconfiguration — not the recipe default. To reproduce that sweep, start from the\ndefault command above and apply these deltas:\n\n- Use `--tensor-parallel-size 4` (the sweep runs TP4; the default recipe keeps\n TP8 for KV-cache and multimodal-encoder headroom).\n- Add `--kv-cache-dtype fp8`.\n- Use the benchmark block size and scheduler knobs: `--block-size 16`,\n `--max-num-batched-tokens 16384`, `--max-num-seqs 512`,\n `--async-scheduling`, and `--no-enable-prefix-caching`.\n- Run on a current `vllm/vllm-openai-rocm:nightly` that contains the required\n AITER Kimi MXFP4 backend.\n- Export the AITER and INT4 quantized all-reduce env vars before launch:\n\n```bash\n# Kernel selection and benchmark runtime knobs.\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\nexport VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1\nexport VLLM_ROCM_USE_SKINNY_GEMM=0\nexport VLLM_ROCM_USE_AITER_RMSNORM=0\nexport AITER_MXFP4_INTERMEDIATE=1\nexport AITER_BYPASS_TUNE_CONFIG=0\nexport AITER_MOE_SORT_BACKEND=auto\nexport OMP_NUM_THREADS=1\n```\n\nThe full reproduction serve command is then:\n\n```bash\nvllm serve amd/Kimi-K2.5-MXFP4 \\\n --port \"${PORT:-8000}\" \\\n --tensor-parallel-size 4 \\\n --gpu-memory-utilization 0.85 \\\n --max-model-len \"$MAX_MODEL_LEN\" \\\n --kv-cache-dtype fp8 \\\n --block-size 16 \\\n --max-num-batched-tokens 16384 \\\n --max-num-seqs 512 \\\n --async-scheduling \\\n --trust-remote-code \\\n --no-enable-prefix-caching \\\n --mm-encoder-tp-mode data\n```\n\nThe benchmark script enforces `MAX_MODEL_LEN >= 9472` so the 1k/1k\nconfiguration matches the locally validated server. The env vars and scheduler\nknobs are benchmark-specific throughput settings, not default serving choices.\n\n#### Tuned AITER MXFP4 MoE (MI355X)\n\nFor higher MoE throughput on MI355X, run the AITER MXFP4 (RadeonFlow)\nintermediate GEMM path with fused shared experts. This requires a\n`vllm/vllm-openai-rocm:nightly` built after vLLM\n[#48683](https://github.com/vllm-project/vllm/pull/48683) (AITER `v0.1.16.post5`,\nwhich includes the [ROCm/aiter#3832](https://github.com/ROCm/aiter/pull/3832)\ngfx950 MXFP4 MoE backend). Export the AITER MoE kernel-selection vars and add the\nbatching/scheduling flags below. `AITER_MXFP4_INTERMEDIATE=1` is what selects the\nRadeonFlow MXFP4 intermediate path — without it (and `VLLM_ROCM_USE_AITER=1`) the\nMoE oracle falls back to a different kernel. Runs at TP4 or TP8; for **TP < 8**\ndisable AITER RMSNorm.\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\nexport VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1\nexport VLLM_ROCM_USE_SKINNY_GEMM=0\nexport AITER_MXFP4_INTERMEDIATE=1\nexport AITER_BYPASS_TUNE_CONFIG=0\nexport AITER_MOE_SORT_BACKEND=auto\nexport OMP_NUM_THREADS=1\n# TP < 8 only:\nexport VLLM_ROCM_USE_AITER_RMSNORM=0\n\nvllm serve amd/Kimi-K2.5-MXFP4 \\\n --tensor-parallel-size 4 \\\n --trust-remote-code \\\n --gpu-memory-utilization 0.90 \\\n --kv-cache-dtype fp8 \\\n --block-size 16 \\\n --async-scheduling \\\n --no-enable-prefix-caching \\\n --mm-encoder-tp-mode data\n```\n\nThese AITER MXFP4 MoE env vars are kernel-selection choices (same MXFP4 math,\nfaster kernel), not precision reductions.\n\nFor fixed-length throughput sweeping, additionally cap `--max-model-len` to the\nworking sequence length (e.g. `9472` for an 8k/1k workload) and set\n`--max-num-batched-tokens 16384 --max-num-seqs 512`. Leave these off for general\nserving.\n\n## Client Usage\n\nOnce the vLLM server is running, consume it via the OpenAI-compatible API:\n\n```python\nimport time\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=\"EMPTY\",\n base_url=\"http://localhost:8000/v1\",\n timeout=3600\n)\n\nmessages = [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": \"https://ofasys-multimodal-wlcb-3-toshanghai.oss-accelerate.aliyuncs.com/wpf272043/keepme/image/receipt.png\"\n }\n },\n {\n \"type\": \"text\",\n \"text\": \"Read all the text in the image.\"\n }\n ]\n }\n]\n\nstart = time.time()\nresponse = client.chat.completions.create(\n model=\"moonshotai/Kimi-K2.5\",\n messages=messages,\n max_tokens=2048\n)\nprint(f\"Response costs: {time.time() - start:.2f}s\")\nprint(f\"Generated text: {response.choices[0].message.content}\")\n```\n\n## Troubleshooting\n\n- **OOM errors:** Lower `--gpu-memory-utilization` or adjust TP/EP to match your GPU count.\n- **Vision encoder performance:** Use `--mm-encoder-tp-mode data` to run the vision encoder\n in data-parallel mode. The encoder is small, so TP adds communication overhead with little gain.\n- **Unique multimodal inputs:** Pass `--mm-processor-cache-gb 0` to avoid caching overhead.\n For repeated inputs, `--mm-processor-cache-type shm` uses host shared memory for better\n performance at high TP settings.\n- **MoE kernel tuning:** Use the `benchmark_moe` script from vLLM to tune Triton kernels\n for your specific hardware.\n- **Async scheduling:** Enabled by default for better throughput. Disable if you encounter\n issues, and file a bug report to vLLM.\n\n## References\n\n- [Kimi-K2.5 on Hugging Face](https://huggingface.co/moonshotai/Kimi-K2.5)\n- [NVIDIA Kimi-K2.5-NVFP4 on Hugging Face](https://huggingface.co/nvidia/Kimi-K2.5-NVFP4)\n- [Eagle3 MLA speculative decoding model](https://huggingface.co/lightseekorg/kimi-k2.5-eagle3-mla)\n- [vLLM multimodal inputs guide](https://docs.vllm.ai/en/latest/features/multimodal_inputs.html)\n- [vLLM Expert Parallelism docs](https://docs.vllm.ai/en/latest/serving/expert_parallel_deployment.html)\n- [vLLM NixlConnector usage guide](https://docs.vllm.ai/en/latest/features/nixl_connector_usage.html)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/moonshotai/Kimi-K2.5.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/moonshotai/Kimi-K2.5.yaml", "overlays": [] } }