{ "hf_id": "moonshotai/Kimi-K2-Instruct", "meta": { "title": "Kimi-K2-Instruct", "slug": "kimi-k2-instruct", "provider": "Moonshot AI", "description": "Moonshot AI's Kimi-K2 is a trillion-parameter MoE instruction model (~32B active) with native FP8 weights and strong tool-calling capabilities.", "date_added": "2025-09-25", "date_updated": "2026-04-17", "tasks": [ "text" ], "performance_headline": "Open-weights 1T-parameter MoE with native FP8 and Kimi K2 tool calling", "related_recipes": [ "moonshotai/Kimi-K2-Thinking", "moonshotai/Kimi-K2.5" ], "hardware": { "h200": "verified", "mi300x": "verified", "mi325x": "verified", "mi355x": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "multi_node_tp", "variant": "default", "node_count": 2, "deploy_type": "multi_node", "env": { "GLOO_SOCKET_IFNAME": "$IFACE_NAME", "NCCL_SOCKET_IFNAME": "$IFACE_NAME" }, "docker_image": "vllm/vllm-openai:latest", "head_command": "vllm serve moonshotai/Kimi-K2-Instruct \\\n --trust-remote-code \\\n --tokenizer-mode auto \\\n --tensor-parallel-size 16 \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr $HEAD_IP \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2", "worker_command": "vllm serve moonshotai/Kimi-K2-Instruct \\\n --trust-remote-code \\\n --tokenizer-mode auto \\\n --tensor-parallel-size 16 \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr $HEAD_IP \\\n --headless \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2", "worker_commands": [ "vllm serve moonshotai/Kimi-K2-Instruct \\\n --trust-remote-code \\\n --tokenizer-mode auto \\\n --tensor-parallel-size 16 \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr $HEAD_IP \\\n --headless \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2" ], "head_argv": [ "vllm", "serve", "moonshotai/Kimi-K2-Instruct", "--trust-remote-code", "--tokenizer-mode", "auto", "--tensor-parallel-size", "16", "--nnodes", "2", "--node-rank", "0", "--master-addr", "$HEAD_IP", "--enable-auto-tool-choice", "--tool-call-parser", "kimi_k2" ], "worker_argv": [ "vllm", "serve", "moonshotai/Kimi-K2-Instruct", "--trust-remote-code", "--tokenizer-mode", "auto", "--tensor-parallel-size", "16", "--nnodes", "2", "--node-rank", "1", "--master-addr", "$HEAD_IP", "--headless", "--enable-auto-tool-choice", "--tool-call-parser", "kimi_k2" ], "worker_argvs": [ [ "vllm", "serve", "moonshotai/Kimi-K2-Instruct", "--trust-remote-code", "--tokenizer-mode", "auto", "--tensor-parallel-size", "16", "--nnodes", "2", "--node-rank", "1", "--master-addr", "$HEAD_IP", "--headless", "--enable-auto-tool-choice", "--tool-call-parser", "kimi_k2" ] ], "head_docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e GLOO_SOCKET_IFNAME=$IFACE_NAME \\\n -e NCCL_SOCKET_IFNAME=$IFACE_NAME \\\n vllm/vllm-openai:latest moonshotai/Kimi-K2-Instruct \\\n --trust-remote-code \\\n --tokenizer-mode auto \\\n --tensor-parallel-size 16 \\\n --nnodes 2 \\\n --node-rank 0 \\\n --master-addr $HEAD_IP \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2", "worker_docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e GLOO_SOCKET_IFNAME=$IFACE_NAME \\\n -e NCCL_SOCKET_IFNAME=$IFACE_NAME \\\n vllm/vllm-openai:latest moonshotai/Kimi-K2-Instruct \\\n --trust-remote-code \\\n --tokenizer-mode auto \\\n --tensor-parallel-size 16 \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr $HEAD_IP \\\n --headless \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2", "worker_docker_commands": [ "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n -e GLOO_SOCKET_IFNAME=$IFACE_NAME \\\n -e NCCL_SOCKET_IFNAME=$IFACE_NAME \\\n vllm/vllm-openai:latest moonshotai/Kimi-K2-Instruct \\\n --trust-remote-code \\\n --tokenizer-mode auto \\\n --tensor-parallel-size 16 \\\n --nnodes 2 \\\n --node-rank 1 \\\n --master-addr $HEAD_IP \\\n --headless \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2" ], "head_docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "GLOO_SOCKET_IFNAME=$IFACE_NAME", "-e", "NCCL_SOCKET_IFNAME=$IFACE_NAME", "vllm/vllm-openai:latest", "moonshotai/Kimi-K2-Instruct", "--trust-remote-code", "--tokenizer-mode", "auto", "--tensor-parallel-size", "16", "--nnodes", "2", "--node-rank", "0", "--master-addr", "$HEAD_IP", "--enable-auto-tool-choice", "--tool-call-parser", "kimi_k2" ], "worker_docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "GLOO_SOCKET_IFNAME=$IFACE_NAME", "-e", "NCCL_SOCKET_IFNAME=$IFACE_NAME", "vllm/vllm-openai:latest", "moonshotai/Kimi-K2-Instruct", "--trust-remote-code", "--tokenizer-mode", "auto", "--tensor-parallel-size", "16", "--nnodes", "2", "--node-rank", "1", "--master-addr", "$HEAD_IP", "--headless", "--enable-auto-tool-choice", "--tool-call-parser", "kimi_k2" ], "worker_docker_argvs": [ [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "-e", "GLOO_SOCKET_IFNAME=$IFACE_NAME", "-e", "NCCL_SOCKET_IFNAME=$IFACE_NAME", "vllm/vllm-openai:latest", "moonshotai/Kimi-K2-Instruct", "--trust-remote-code", "--tokenizer-mode", "auto", "--tensor-parallel-size", "16", "--nnodes", "2", "--node-rank", "1", "--master-addr", "$HEAD_IP", "--headless", "--enable-auto-tool-choice", "--tool-call-parser", "kimi_k2" ] ], "strategy_spec": { "name": "multi_node_tp", "deploy_type": "multi_node", "display_name": "Multi-Node Tensor Parallel", "short_label": "Multi-node TP", "orientation": "latency", "description": "Multi-node TP. Splits the model across GPUs spanning multiple nodes. TP is set to total GPU count across all nodes. Head node serves requests, worker nodes run with --headless. Requires InfiniBand interconnect.\n", "hardware_match": { "min_gpus": 2, "multi_node": true }, "parallel_flag": "--tensor-parallel-size", "env": { "GLOO_SOCKET_IFNAME": "$IFACE_NAME", "NCCL_SOCKET_IFNAME": "$IFACE_NAME" } }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e ยท 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tp": "/moonshotai/Kimi-K2-Instruct/strategies/single_node_tp.json", "single_node_tep": "/moonshotai/Kimi-K2-Instruct/strategies/single_node_tep.json", "single_node_dep": "/moonshotai/Kimi-K2-Instruct/strategies/single_node_dep.json", "multi_node_tp_pp": "/moonshotai/Kimi-K2-Instruct/strategies/multi_node_tp_pp.json", "multi_node_dep": "/moonshotai/Kimi-K2-Instruct/strategies/multi_node_dep.json", "multi_node_tep": "/moonshotai/Kimi-K2-Instruct/strategies/multi_node_tep.json", "pd_cluster": "/moonshotai/Kimi-K2-Instruct/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/moonshotai/Kimi-K2-Instruct/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/moonshotai/Kimi-K2-Instruct/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/moonshotai/Kimi-K2-Instruct/hw/h200.json", "h100": "/moonshotai/Kimi-K2-Instruct/hw/h100.json", "b200": "/moonshotai/Kimi-K2-Instruct/hw/b200.json", "gb200": "/moonshotai/Kimi-K2-Instruct/hw/gb200.json", "b300": "/moonshotai/Kimi-K2-Instruct/hw/b300.json", "gb300": "/moonshotai/Kimi-K2-Instruct/hw/gb300.json", "mi300x": "/moonshotai/Kimi-K2-Instruct/hw/mi300x.json", "mi325x": "/moonshotai/Kimi-K2-Instruct/hw/mi325x.json", "mi350x": "/moonshotai/Kimi-K2-Instruct/hw/mi350x.json", "mi355x": "/moonshotai/Kimi-K2-Instruct/hw/mi355x.json" } }, "model": { "model_id": "moonshotai/Kimi-K2-Instruct", "min_vllm_version": "0.12.0", "architecture": "moe", "parameter_count": "1T", "active_parameters": "32B", "context_length": 131072, "supports_dcp": true, "base_args": [ "--trust-remote-code", "--tokenizer-mode", "auto" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" }, "extras": [ { "note": "Optional: DeepEP + DeepGEMM for the DP+EP deployment path on H800/H200", "command": "uv pip install git+https://github.com/deepseek-ai/DeepGEMM.git@v2.1.1.post3 --no-build-isolation", "optional": true } ] } }, "features": { "tool_calling": { "description": "Enable Kimi K2 tool calling with the kimi_k2 tool-call parser.", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "kimi_k2" ] } }, "opt_in_features": [], "variants": { "default": { "precision": "fp8", "vram_minimum_gb": 1200, "description": "Native FP8 weights on 16xH800 / 16xH200 (smallest deployment for 128k seqlen)" } }, "hardware_overrides": { "amd": { "extra_env": { "SAFETENSORS_FAST_GPU": "1" } } }, "guide": "## Overview\n\nKimi-K2-Instruct is Moonshot AI's trillion-parameter Mixture-of-Experts instruction\nmodel (approximately 32B activated per token) shipped with native FP8 weights. The\nsmallest deployment unit for Kimi-K2 FP8 weights with 128k seqlen on mainstream H800\nplatforms is a 16-GPU cluster using either Tensor Parallel (TP) or Data Parallel +\nExpert Parallel (DP+EP). This guide is partially adapted from the official\n[Kimi-K2-Instruct Deployment Guidance](https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/main/docs/deploy_guidance.md).\n\n## Prerequisites\n\n- **Hardware (FP8)**: 16x H800 or 16x H200 GPUs (verified)\n- **vLLM**: Current stable release\n\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto\n```\n\n### AMD ROCm\n\nThe ROCm wheel requires Python 3.12, ROCm 7.0+, and glibc >= 2.35. Use the\nROCm Docker image when your host environment does not meet those requirements.\n\n```bash\nuv venv --python 3.12\nsource .venv/bin/activate\nuv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/\n```\n\n## Running Kimi-K2 with FP8 on 16xH800\n\n### Tensor Parallelism + Pipeline Parallelism (TP8+PP2)\n\n```bash\n# node 0 (start Ray on both nodes first)\nvllm serve moonshotai/Kimi-K2-Instruct \\\n --trust-remote-code \\\n --tokenizer-mode auto \\\n --tensor-parallel-size 8 \\\n --pipeline-parallel-size 2 \\\n --dtype bfloat16 \\\n --quantization fp8 \\\n --max-model-len 2048 \\\n --max-num-seqs 1 \\\n --max-num-batched-tokens 1024 \\\n --enable-chunked-prefill \\\n --kv-cache-dtype fp8 \\\n -dcp 8\n```\n\nKey parameter notes:\n- `--enable-auto-tool-choice`: required when enabling tool usage.\n- `--tool-call-parser kimi_k2`: required when enabling tool usage.\n\n### Data Parallelism + Expert Parallelism (DP16+EP)\n\nYou can install libraries like DeepEP and DeepGEMM as needed. Then run the command\n(example on H800):\n\n```bash\n# node 0\nvllm serve moonshotai/Kimi-K2-Instruct \\\n --port 8000 --served-model-name kimi-k2 \\\n --trust-remote-code \\\n --data-parallel-size 16 \\\n --data-parallel-size-local 8 \\\n --data-parallel-address $MASTER_IP \\\n --data-parallel-rpc-port $PORT \\\n --enable-expert-parallel \\\n --max-num-batched-tokens 8192 \\\n --max-num-seqs 256 \\\n --gpu-memory-utilization 0.85 \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2\n\n# node 1\nvllm serve moonshotai/Kimi-K2-Instruct \\\n --headless \\\n --data-parallel-start-rank 8 \\\n --port 8000 --served-model-name kimi-k2 \\\n --trust-remote-code \\\n --data-parallel-size 16 \\\n --data-parallel-size-local 8 \\\n --data-parallel-address $MASTER_IP \\\n --data-parallel-rpc-port $PORT \\\n --enable-expert-parallel \\\n --max-num-batched-tokens 8192 \\\n --max-num-seqs 256 \\\n --gpu-memory-utilization 0.85 \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2\n```\n\n### ROCm on 8x MI300X / MI325X / MI355X\n\n```bash\nexport SAFETENSORS_FAST_GPU=1\nvllm serve moonshotai/Kimi-K2-Instruct \\\n --tensor-parallel-size 8 \\\n --trust-remote-code \\\n --enable-auto-tool-choice \\\n --tool-call-parser kimi_k2\n```\n\nAdditional flags:\n- `--max-model-len` preserves memory; `--max-model-len=65536` is usually good for most scenarios.\n- `--max-num-batched-tokens` balances throughput vs latency. `32768` is good for prompt-heavy\n workloads; reduce to 16k or 8k to cut activation memory and decrease latency.\n- vLLM conservatively uses 90% of GPU memory. Set `--gpu-memory-utilization=0.95` to\n maximize KV cache.\n\n## Benchmarking\n\n### FP8 Benchmark on 16xH800\n\n```bash\nvllm bench serve \\\n --model moonshotai/Kimi-K2-Instruct \\\n --dataset-name random \\\n --random-input-len 1000 \\\n --random-output-len 512 \\\n --request-rate 1.0 \\\n --num-prompts 8 \\\n --ignore-eos \\\n --trust-remote-code\n```\n\n### FP8 Benchmark on 16xH200\n\n```bash\nvllm bench serve \\\n --model moonshotai/Kimi-K2-Instruct \\\n --dataset-name random \\\n --random-input-len 8000 \\\n --random-output-len 1000 \\\n --request-rate 10000 \\\n --num-prompts 16 \\\n --ignore-eos \\\n --trust-remote-code\n```\n\nAdding `-dcp 8` at launch can further improve throughput on H200 (observed ~33% lower\nmean TTFT and higher tok/s in internal benchmarks).\n\nTest different batch sizes by changing `--num-prompts`, e.g. 1, 16, 32, 64, 128, 256, 512.\n\n## References\n\n- [Kimi-K2-Instruct on Hugging Face](https://huggingface.co/moonshotai/Kimi-K2-Instruct)\n- [Official Kimi-K2 Deployment Guidance](https://huggingface.co/moonshotai/Kimi-K2-Instruct/blob/main/docs/deploy_guidance.md)\n- [vLLM Expert Parallelism docs](https://docs.vllm.ai/en/latest/serving/expert_parallel_deployment.html)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/moonshotai/Kimi-K2-Instruct.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/moonshotai/Kimi-K2-Instruct.yaml", "overlays": [] } }