{ "hf_id": "stepfun-ai/Step-3.5-Flash", "meta": { "title": "Step-3.5-Flash", "slug": "step-3.5-flash", "provider": "StepFun", "description": "Production-grade reasoning MoE (~196B total / 11B active parameters) with hybrid attention schedules, SWA compensation, and multi-token prediction for low-latency long-context inference", "date_added": "2026-02-01", "date_updated": "2026-04-17", "tasks": [ "text" ], "performance_headline": "Sparse MoE reasoning model with hybrid attention and step3p5 MTP speculative decoding", "related_recipes": [], "hardware": { "h200": "verified", "b200": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:latest", "command": "vllm serve stepfun-ai/Step-3.5-Flash \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser step3p5 \\\n --enable-auto-tool-choice \\\n --reasoning-parser step3p5", "argv": [ "vllm", "serve", "stepfun-ai/Step-3.5-Flash", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "step3p5", "--enable-auto-tool-choice", "--reasoning-parser", "step3p5" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:latest stepfun-ai/Step-3.5-Flash \\\n --trust-remote-code \\\n --tensor-parallel-size 8 \\\n --tool-call-parser step3p5 \\\n --enable-auto-tool-choice \\\n --reasoning-parser step3p5", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:latest", "stepfun-ai/Step-3.5-Flash", "--trust-remote-code", "--tensor-parallel-size", "8", "--tool-call-parser", "step3p5", "--enable-auto-tool-choice", "--reasoning-parser", "step3p5" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "single_node_tep": "/stepfun-ai/Step-3.5-Flash/strategies/single_node_tep.json", "single_node_dep": "/stepfun-ai/Step-3.5-Flash/strategies/single_node_dep.json", "multi_node_tp": "/stepfun-ai/Step-3.5-Flash/strategies/multi_node_tp.json", "multi_node_tp_pp": "/stepfun-ai/Step-3.5-Flash/strategies/multi_node_tp_pp.json", "multi_node_tep": "/stepfun-ai/Step-3.5-Flash/strategies/multi_node_tep.json", "multi_node_dep": "/stepfun-ai/Step-3.5-Flash/strategies/multi_node_dep.json", "pd_cluster": "/stepfun-ai/Step-3.5-Flash/strategies/pd_cluster.json", "kv_store_centralized_mooncake": "/stepfun-ai/Step-3.5-Flash/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/stepfun-ai/Step-3.5-Flash/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/stepfun-ai/Step-3.5-Flash/hw/h200.json", "h100": "/stepfun-ai/Step-3.5-Flash/hw/h100.json", "b200": "/stepfun-ai/Step-3.5-Flash/hw/b200.json", "gb200": "/stepfun-ai/Step-3.5-Flash/hw/gb200.json", "b300": "/stepfun-ai/Step-3.5-Flash/hw/b300.json", "gb300": "/stepfun-ai/Step-3.5-Flash/hw/gb300.json", "mi300x": "/stepfun-ai/Step-3.5-Flash/hw/mi300x.json", "mi325x": "/stepfun-ai/Step-3.5-Flash/hw/mi325x.json", "mi350x": "/stepfun-ai/Step-3.5-Flash/hw/mi350x.json", "mi355x": "/stepfun-ai/Step-3.5-Flash/hw/mi355x.json" } }, "model": { "model_id": "stepfun-ai/Step-3.5-Flash", "min_vllm_version": "0.15.1", "architecture": "moe", "parameter_count": "196B", "active_parameters": "11B", "context_length": 262144, "base_args": [ "--trust-remote-code" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:latest" } } }, "features": { "tool_calling": { "description": "Step-3.5 tool call parser with automatic tool choice", "args": [ "--tool-call-parser", "step3p5", "--enable-auto-tool-choice" ] }, "reasoning": { "description": "Step-3.5 reasoning parser for chain-of-thought extraction", "args": [ "--reasoning-parser", "step3p5" ] }, "spec_decoding": { "description": "Multi-Token Prediction speculative decoding with the step3p5_mtp method", "args": [ "--hf-overrides", "{\"num_nextn_predict_layers\": 1}", "--speculative-config", "{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}" ] } }, "opt_in_features": [ "spec_decoding" ], "variants": { "default": { "precision": "bf16", "vram_minimum_gb": 470, "description": "Full precision BF16 — runs on 4xH200/H20/B200" }, "fp8": { "model_id": "stepfun-ai/Step-3.5-Flash-FP8", "precision": "fp8", "vram_minimum_gb": 235, "tp": 2, "description": "Native FP8 checkpoint (TP not supported beyond 2 — use DP4)", "json": "/stepfun-ai/Step-3.5-Flash-FP8.json" }, "int4": { "model_id": "stepfun-ai/Step-3.5-Flash-INT4", "precision": "int4", "vram_minimum_gb": 118, "description": "INT4 quantized weights", "json": "/stepfun-ai/Step-3.5-Flash-INT4.json" }, "int8": { "model_id": "stepfun-ai/Step-3.5-Flash-INT8", "precision": "int8", "vram_minimum_gb": 235, "description": "INT8 quantized weights", "json": "/stepfun-ai/Step-3.5-Flash-INT8.json" } }, "hardware_overrides": { "amd": { "extra_args": [], "extra_env": { "VLLM_ROCM_USE_AITER": "1", "VLLM_ROCM_USE_AITER_MOE": "0" } } }, "guide": "## Overview\n\n[Step-3.5-Flash](https://huggingface.co/stepfun-ai/Step-3.5-Flash) is an advanced\nreasoning engine from [StepFun](https://www.stepfun.com/company). Highlights:\n\n- Hybrid attention schedules with compensation for sliding-window attention (SWA)\n- Sparse MoE structure (196B total parameters, 11B active)\n- Multi-token prediction mechanism for faster inference\n\nAvailable precisions:\n\n- [stepfun-ai/Step-3.5-Flash](https://huggingface.co/stepfun-ai/Step-3.5-Flash) (BF16)\n- [stepfun-ai/Step-3.5-Flash-FP8](https://huggingface.co/stepfun-ai/Step-3.5-Flash-FP8)\n- [stepfun-ai/Step-3.5-Flash-Int4](https://huggingface.co/stepfun-ai/Step-3.5-Flash-Int4) (not yet supported by vLLM)\n\n## Prerequisites\n\n- **vLLM version:** latest stable\n- **Hardware:** 4x H200/H20/B200\n\n### Install vLLM\n\n```bash\nuv venv\nsource .venv/bin/activate\nuv pip install vllm --torch-backend auto\n```\n\n## Launching the Server\n\n### Tensor Parallel\n\n```bash\nvllm serve stepfun-ai/Step-3.5-Flash \\\n --tensor-parallel-size 4 \\\n --reasoning-parser step3p5 \\\n --tool-call-parser step3p5 \\\n --enable-auto-tool-choice \\\n --trust-remote-code\n```\n\nNote: The FP8 version cannot use TP4 — use DP4 instead.\n\n### Data Parallel + Expert Parallel (recommended for FP8)\n\n```bash\nvllm serve stepfun-ai/Step-3.5-Flash \\\n --data-parallel-size 4 \\\n --enable-expert-parallel \\\n --reasoning-parser step3p5 \\\n --tool-call-parser step3p5 \\\n --enable-auto-tool-choice \\\n --trust-remote-code\n```\n\n### Enabling MTP Speculative Decoding\n\n```bash\nvllm serve stepfun-ai/Step-3.5-Flash \\\n --tensor-parallel-size 4 \\\n --reasoning-parser step3p5 \\\n --tool-call-parser step3p5 \\\n --enable-auto-tool-choice \\\n --trust-remote-code \\\n --hf-overrides '{\"num_nextn_predict_layers\": 1}' \\\n --speculative-config '{\"method\": \"step3p5_mtp\", \"num_speculative_tokens\": 1}'\n```\n\n## Benchmarking\n\n```bash\nvllm bench serve \\\n --backend vllm \\\n --model stepfun-ai/Step-3.5-Flash \\\n --endpoint /v1/completions \\\n --dataset-name random \\\n --random-input 2048 \\\n --random-output 1024 \\\n --max-concurrency 10 \\\n --num-prompt 100\n```\n\n## Troubleshooting\n\n- **MoE kernel tuning:** See [tune-moe-kernel](https://github.com/vllm-project/recipes/blob/main/Qwen/Qwen3-Next.md#tune-moe-kernel)\n to tune Triton kernels for your hardware.\n- **FP8 DeepGEMM:** For FP8, install DeepGEMM via [install_deepgemm.sh](https://github.com/vllm-project/vllm/blob/v0.16.0rc0/tools/install_deepgemm.sh).\n- **B200 FlashInfer FP8 MoE error:** The `routing_logits must be bfloat16`\n error on B200 was fixed in vLLM v0.28.0 — no workaround is needed.\n- **FP8 + TP4 incompatibility:** Use DP4+EP instead.\n\n## References\n\n- [Model card](https://huggingface.co/stepfun-ai/Step-3.5-Flash)\n- [FP8 checkpoint](https://huggingface.co/stepfun-ai/Step-3.5-Flash-FP8)\n- [StepFun](https://www.stepfun.com/company)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/stepfun-ai/Step-3.5-Flash.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/stepfun-ai/Step-3.5-Flash.yaml", "overlays": [] } }