{ "hf_id": "moonshotai/Kimi-K2.6", "meta": { "title": "Kimi-K2.6", "slug": "kimi-k2.6", "provider": "Moonshot AI", "description": "Open-source native multimodal agentic MoE model with vision-language understanding, tool calling, and thinking modes", "date_added": "2026-04-20", "date_updated": "2026-07-29", "tasks": [ "multimodal", "text" ], "performance_headline": "Multimodal agentic MoE model with DeepSeek-V3 backbone and MLA attention", "related_recipes": [], "hardware": { "h200": "verified", "b300": "verified", "gb200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve moonshotai/Kimi-K2.6 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser kimi_k2 \\\n --enable-auto-tool-choice \\\n --reasoning-parser kimi_k2 \\\n --speculative-config '{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}' \\\n --mm-encoder-tp-mode data", "argv": [ "vllm", "serve", "moonshotai/Kimi-K2.6", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "kimi_k2", "--enable-auto-tool-choice", "--reasoning-parser", "kimi_k2", "--speculative-config", "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--mm-encoder-tp-mode", "data" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest moonshotai/Kimi-K2.6 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser kimi_k2 \\\n --enable-auto-tool-choice \\\n --reasoning-parser kimi_k2 \\\n --speculative-config '{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}' \\\n --mm-encoder-tp-mode data", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "moonshotai/Kimi-K2.6", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "kimi_k2", "--enable-auto-tool-choice", "--reasoning-parser", "kimi_k2", "--speculative-config", "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}", "--mm-encoder-tp-mode", "data" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/moonshotai/Kimi-K2.6/strategies/single_node_tep.json", "single_node_dep": "/moonshotai/Kimi-K2.6/strategies/single_node_dep.json", "multi_node_tp": "/moonshotai/Kimi-K2.6/strategies/multi_node_tp.json", "multi_node_tp_pp": "/moonshotai/Kimi-K2.6/strategies/multi_node_tp_pp.json", "multi_node_dep": "/moonshotai/Kimi-K2.6/strategies/multi_node_dep.json", "multi_node_tep": "/moonshotai/Kimi-K2.6/strategies/multi_node_tep.json", "pd_cluster": "/moonshotai/Kimi-K2.6/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/moonshotai/Kimi-K2.6/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/moonshotai/Kimi-K2.6/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/moonshotai/Kimi-K2.6/hw/h200.json", "h100": "/moonshotai/Kimi-K2.6/hw/h100.json", "b200": "/moonshotai/Kimi-K2.6/hw/b200.json", "gb200": "/moonshotai/Kimi-K2.6/hw/gb200.json", "b300": "/moonshotai/Kimi-K2.6/hw/b300.json", "gb300": "/moonshotai/Kimi-K2.6/hw/gb300.json", "mi300x": "/moonshotai/Kimi-K2.6/hw/mi300x.json", "mi325x": "/moonshotai/Kimi-K2.6/hw/mi325x.json", "mi350x": "/moonshotai/Kimi-K2.6/hw/mi350x.json", "mi355x": "/moonshotai/Kimi-K2.6/hw/mi355x.json" } }, "model": { "model_id": "moonshotai/Kimi-K2.6", "min_vllm_version": "0.25.0", "docker_image": { "nvidia": "vllm/vllm-openai:latest", "amd": "vllm/vllm-openai-rocm:nightly" }, "architecture": "moe", "parameter_count": "1T", "active_parameters": "32B", "context_length": 262144, "supports_dcp": true, "base_args": [ "--trust-remote-code" ], "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto", "note": "The optimized B300 EAGLE3 and native CPU KV offload path requires vLLM >= 0.25.0." }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Kimi K2 tool-call parser with automatic tool choice", "args": [ "--tool-call-parser", "kimi_k2", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "Kimi K2 reasoning parser for extracting chain-of-thought content", "args": [ "--reasoning-parser", "kimi_k2" ] }, "spec_decoding": { "description": "Eagle3 speculative decoding for accelerated inference", "args": [ "--speculative-config", "{\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"method\":\"eagle3\",\"num_speculative_tokens\":3}" ] }, "text_only": { "description": "Skip loading the vision encoder for text-only workloads — frees VRAM for KV cache. Mutually exclusive with encoder_parallel.", "args": [ "--language-model-only" ] }, "encoder_parallel": { "description": "Run the vision encoder in data-parallel mode — avoids TP comm overhead on the small encoder. Mutually exclusive with text_only.", "args": [ "--mm-encoder-tp-mode", "data" ] } }, "opt_in_features": [ "text_only" ], "hardware_opt_in_features": { "gb200": [ "encoder_parallel" ] }, "variants": { "default": { "precision": "int4", "vram_minimum_gb": 714, "description": "Packed INT4 via compressed-tensors (~595 GB on disk); fits 8×H200" }, "nvfp4": { "model_id": "nvidia/Kimi-K2.6-NVFP4", "precision": "nvfp4", "vram_minimum_gb": 600, "description": "NVIDIA NVFP4 quantized weights for Blackwell GPUs (e.g. GB200)", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/nvidia/Kimi-K2.6-NVFP4.json" }, "fp8": { "model_id": "RedHatAI/Kimi-K2.6-FP8-BLOCK", "precision": "fp8", "vram_minimum_gb": 1200, "description": "RedHatAI FP8 block quantization", "extra_args": [ "--kv-cache-dtype", "fp8" ], "json": "/RedHatAI/Kimi-K2.6-FP8-BLOCK.json" }, "redhat_nvfp4": { "model_id": "RedHatAI/Kimi-K2.6-NVFP4", "precision": "nvfp4", "label": "NVFP4 (RedHat)", "vram_minimum_gb": 600, "description": "RedHatAI NVFP4 quantization", "json": "/RedHatAI/Kimi-K2.6-NVFP4.json" } }, "hardware_overrides": { "blackwell": { "extra_args": [ "--attention-config", "{\"use_prefill_query_quantization\":true}" ] }, "amd": { "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_QUICK_REDUCE_QUANTIZATION": "INT4" } } }, "guide": "## Overview\n\nKimi K2.6 is an open-source, native multimodal agentic model built through continual\npretraining on approximately 15 trillion mixed visual and text tokens atop Kimi-K2-Base.\nIt seamlessly integrates vision and language understanding with advanced agentic\ncapabilities, instant and thinking modes, as well as conversational and agentic paradigms.\n\n## Prerequisites\n\n- **vLLM version:** >= 0.25.0 (required for the B300 EAGLE3 and native CPU KV offload paths below)\n- **Hardware (INT4):** 8× H200 GPUs (verified), or equivalent aggregate VRAM (~640 GB)\n- **Hardware (B300 / NVFP4):** 4× B300 (verified); use `vllm/vllm-openai:latest`\n- **AMD support:** 8x MI300X / MI325X / MI355X with ROCm 7.2.1 and Python 3.12\n\n### NVIDIA B300: NVFP4 with Eagle3\n\n```bash\nexport VLLM_FLASHINFER_ALLREDUCE_BACKEND=trtllm\n\nvllm serve nvidia/Kimi-K2.6-NVFP4 \\\n --tensor-parallel-size 4 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 64 \\\n --gpu-memory-utilization 0.90 \\\n --attention-backend TOKENSPEED_MLA \\\n --attention-config '{\"use_prefill_query_quantization\":true}' \\\n --compilation-config '{\"cudagraph_mode\":\"FULL_AND_PIECEWISE\",\"custom_ops\":[\"all\"]}' \\\n --max-cudagraph-capture-size 2048 \\\n --speculative-config '{\"method\":\"eagle3\",\"model\":\"lightseekorg/kimi-k2.6-eagle3-mla\",\"num_speculative_tokens\":4}'\n```\n\n### Native CPU KV offload\n\nSelect **Simple** in the command builder's **KV Offload** row to extend the prefix cache\ninto host DRAM with `SimpleCPUOffloadConnector`. The shared option uses 220 GiB per rank\nby default. The verified B300 TP4 run used 1,199 GiB total (299.75 GiB per rank); the\nequivalent explicit command is:\n\n```bash\nCPU_OFFLOAD_BYTES_PER_RANK=321854111744\n\nvllm serve nvidia/Kimi-K2.6-NVFP4 \\\n --tensor-parallel-size 4 \\\n --trust-remote-code \\\n --kv-cache-dtype fp8 \\\n --block-size 64 \\\n --gpu-memory-utilization 0.90 \\\n --attention-backend TOKENSPEED_MLA \\\n --attention-config '{\"use_prefill_query_quantization\":true}' \\\n --disable-hybrid-kv-cache-manager \\\n --kv-transfer-config \"{\\\"kv_connector\\\":\\\"SimpleCPUOffloadConnector\\\",\\\"kv_role\\\":\\\"kv_both\\\",\\\"kv_connector_extra_config\\\":{\\\"cpu_bytes_to_use_per_rank\\\":${CPU_OFFLOAD_BYTES_PER_RANK},\\\"lazy_offload\\\":false}}\"\n```\n\n### Decode context parallelism\n\nFor higher concurrency on the B300 NVFP4 path, add `--decode-context-parallel-size 4`\n(match TP size). The command builder's **Advanced** row also exposes DCP for this recipe.\n\n### AMD MI300X/MI325X\n\nOn 8x MI300X or MI325X (`gfx942`), use the standard W4A16 MoE path with AITER\nand INT4 QuickReduce.\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\n\nvllm serve moonshotai/Kimi-K2.6 \\\n --host 0.0.0.0 \\\n --port 8000 \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser kimi_k2 \\\n --enable-auto-tool-choice \\\n --reasoning-parser kimi_k2 \\\n --mm-encoder-tp-mode data\n```\n\n### AMD MI350X/MI355X\n\nOn 8x MI350X or MI355X (`gfx950`), add `--moe-backend flydsl` to use the\noptimized FlyDSL W4A16 MoE kernel. Keep LoRA disabled for this path.\n\n```bash\nexport VLLM_ROCM_USE_AITER=1\nexport VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4\n\nvllm serve moonshotai/Kimi-K2.6 \\\n --tensor-parallel-size 8 \\\n --trust-remote-code \\\n --mm-encoder-tp-mode data \\\n --moe-backend flydsl \\\n --compilation-config '{\"pass_config\": {\"fuse_allreduce_rms\": false}}'\n```\n\nNotes:\n- The FlyDSL INT4 MoE path does not support expert parallelism; do not add\n `--enable-expert-parallel`.\n- Keep `--compilation-config '{\"pass_config\": {\"fuse_allreduce_rms\": false}}'`;\n it is required for this FlyDSL path on MI350X / MI355X.\n- vLLM has tuned MI350X/MI355X FlyDSL configs for this Kimi shape at TP=8 and TP=4.\n- Keep vLLM's default block size unless you are tuning long-context\n throughput; `--block-size 64` is safe to try.\n\n## Client Usage\n\nOnce the vLLM server is running, consume it via the OpenAI-compatible API:\n\n```python\nimport time\nfrom openai import OpenAI\n\nclient = OpenAI(\n api_key=\"EMPTY\",\n base_url=\"http://localhost:8000/v1\",\n timeout=3600\n)\n\nmessages = [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"image_url\",\n \"image_url\": {\n \"url\": \"https://ofasys-multimodal-wlcb-3-toshanghai.oss-accelerate.aliyuncs.com/wpf272043/keepme/image/receipt.png\"\n }\n },\n {\n \"type\": \"text\",\n \"text\": \"Read all the text in the image.\"\n }\n ]\n }\n]\n\nstart = time.time()\nresponse = client.chat.completions.create(\n model=\"moonshotai/Kimi-K2.6\",\n messages=messages,\n max_tokens=2048\n)\nprint(f\"Response costs: {time.time() - start:.2f}s\")\nprint(f\"Generated text: {response.choices[0].message.content}\")\n```\n\n## Troubleshooting\n\n- **OOM errors:** Lower `--gpu-memory-utilization` or adjust TP/EP to match your GPU count.\n- **Vision encoder performance:** Use `--mm-encoder-tp-mode data` to run the vision encoder\n in data-parallel mode. The encoder is small, so TP adds communication overhead with little gain.\n- **Unique multimodal inputs:** Pass `--mm-processor-cache-gb 0` to avoid caching overhead.\n For repeated inputs, `--mm-processor-cache-type shm` uses host shared memory for better\n performance at high TP settings.\n- **MoE kernel tuning:** Use the `benchmark_moe` script from vLLM to tune Triton kernels\n for your specific hardware.\n- **Async scheduling:** Enabled by default for better throughput. Disable if you encounter\n issues, and file a bug report to vLLM.\n\n## References\n\n- [Kimi-K2.6 on Hugging Face](https://huggingface.co/moonshotai/Kimi-K2.6)\n- [NVIDIA Kimi-K2.6-NVFP4 on Hugging Face](https://huggingface.co/nvidia/Kimi-K2.6-NVFP4)\n- [vLLM multimodal inputs guide](https://docs.vllm.ai/en/latest/features/multimodal_inputs.html)\n- [vLLM Expert Parallelism docs](https://docs.vllm.ai/en/latest/serving/expert_parallel_deployment.html)\n- [vLLM NixlConnector usage guide](https://docs.vllm.ai/en/latest/features/nixl_connector_usage.html)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/moonshotai/Kimi-K2.6.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/moonshotai/Kimi-K2.6.yaml", "overlays": [] } }