{ "hf_id": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", "meta": { "title": "NVIDIA Nemotron 3.5 Super VL", "slug": "nemotron-3-5-super-vl", "provider": "NVIDIA", "description": "120B total / 12B active vision-language model with NVFP4, FP8, and BF16 recipes for H100, H200, GB200, DGX Spark, and RTX Pro 6000", "date_added": "2026-10-15", "date_updated": "2026-10-15", "tasks": [ "multimodal" ], "related_recipes": [ "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" ], "default_hardware": "h200", "hardware": { "h100": "verified", "h200": "verified", "gb200": "verified", "dgx_spark_gb10": "verified", "rtx_pro_6000": "verified" } }, "recommended_command": { "hardware": "h200", "strategy": "single_node_tp", "variant": "default", "node_count": 1, "deploy_type": "single_node", "env": {}, "docker_image": "vllm/vllm-openai:v0.31.0", "command": "vllm serve nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4 \\\n --trust-remote-code \\\n --tensor-parallel-size 1 \\\n --max-model-len 114688 \\\n --max-num-seqs 128 \\\n --gpu-memory-utilization 0.89 \\\n --max-num-batched-tokens 16384 \\\n --block-size 64 \\\n --prefix-match-unit 64 \\\n --compilation_config.max_cudagraph_capture_size 256 \\\n --mamba-backend flashinfer \\\n --mamba-ssm-cache-dtype float16 \\\n --enable-mamba-cache-stochastic-rounding \\\n --mamba-cache-philox-rounds 5 \\\n --enable-mamba-shared-prefix-checkpoint \\\n --mamba-cache-mode align \\\n --video-pruning-rate 0.5 \\\n --media-io-kwargs '{\"video\":{\"fps\":2,\"num_frames\":256}}' \\\n --speculative-config.method mtp \\\n --speculative-config.num_speculative_tokens 3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser nemotron_v3", "argv": [ "vllm", "serve", "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", "--trust-remote-code", "--tensor-parallel-size", "1", "--max-model-len", "114688", "--max-num-seqs", "128", "--gpu-memory-utilization", "0.89", "--max-num-batched-tokens", "16384", "--block-size", "64", "--prefix-match-unit", "64", "--compilation_config.max_cudagraph_capture_size", "256", "--mamba-backend", "flashinfer", "--mamba-ssm-cache-dtype", "float16", "--enable-mamba-cache-stochastic-rounding", "--mamba-cache-philox-rounds", "5", "--enable-mamba-shared-prefix-checkpoint", "--mamba-cache-mode", "align", "--video-pruning-rate", "0.5", "--media-io-kwargs", "{\"video\":{\"fps\":2,\"num_frames\":256}}", "--speculative-config.method", "mtp", "--speculative-config.num_speculative_tokens", "3", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "nemotron_v3" ], "docker_command": "docker run --gpus all \\\n --privileged --ipc=host -p 8000:8000 \\\n -v ~/.cache/huggingface:/root/.cache/huggingface \\\n vllm/vllm-openai:v0.31.0 nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4 \\\n --trust-remote-code \\\n --tensor-parallel-size 1 \\\n --max-model-len 114688 \\\n --max-num-seqs 128 \\\n --gpu-memory-utilization 0.89 \\\n --max-num-batched-tokens 16384 \\\n --block-size 64 \\\n --prefix-match-unit 64 \\\n --compilation_config.max_cudagraph_capture_size 256 \\\n --mamba-backend flashinfer \\\n --mamba-ssm-cache-dtype float16 \\\n --enable-mamba-cache-stochastic-rounding \\\n --mamba-cache-philox-rounds 5 \\\n --enable-mamba-shared-prefix-checkpoint \\\n --mamba-cache-mode align \\\n --video-pruning-rate 0.5 \\\n --media-io-kwargs '{\"video\":{\"fps\":2,\"num_frames\":256}}' \\\n --speculative-config.method mtp \\\n --speculative-config.num_speculative_tokens 3 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser nemotron_v3", "docker_argv": [ "docker", "run", "--gpus", "all", "--privileged", "--ipc=host", "-p", "8000:8000", "-v", "~/.cache/huggingface:/root/.cache/huggingface", "vllm/vllm-openai:v0.31.0", "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", "--trust-remote-code", "--tensor-parallel-size", "1", "--max-model-len", "114688", "--max-num-seqs", "128", "--gpu-memory-utilization", "0.89", "--max-num-batched-tokens", "16384", "--block-size", "64", "--prefix-match-unit", "64", "--compilation_config.max_cudagraph_capture_size", "256", "--mamba-backend", "flashinfer", "--mamba-ssm-cache-dtype", "float16", "--enable-mamba-cache-stochastic-rounding", "--mamba-cache-philox-rounds", "5", "--enable-mamba-shared-prefix-checkpoint", "--mamba-cache-mode", "align", "--video-pruning-rate", "0.5", "--media-io-kwargs", "{\"video\":{\"fps\":2,\"num_frames\":256}}", "--speculative-config.method", "mtp", "--speculative-config.num_speculative_tokens", "3", "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder", "--reasoning-parser", "nemotron_v3" ], "strategy_spec": { "name": "single_node_tp", "deploy_type": "single_node", "display_name": "Tensor Parallel", "short_label": "TP", "orientation": "latency", "description": "Single-node tensor parallel. Splits the model across all local GPUs. TP size is set to the GPU count at deploy time. The simplest multi-GPU strategy — works for all model architectures.\n", "hardware_match": { "min_gpus": 1, "max_gpus": 8, "multi_node": false }, "vllm_args": [], "parallel_flag": "--tensor-parallel-size" }, "hardware_profile": { "brand": "NVIDIA", "generation": "hopper", "compute_arch": "sm90", "display_name": "H200", "description": "NVIDIA H200 SXM 141 GB HBM3e · 8-GPU HGX node", "gpu_count": 8, "vram_gb": 1128, "multi_node": false }, "alternatives": { "kv_store_centralized_mooncake": "/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/strategies/kv_store_centralized_mooncake.json", "kv_store_distributed_mooncake": "/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/strategies/kv_store_distributed_mooncake.json" }, "by_hardware": { "h200": "/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/hw/h200.json", "h100": "/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/hw/h100.json", "gb200": "/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/hw/gb200.json", "dgx_spark_gb10": "/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/hw/dgx_spark_gb10.json", "rtx_pro_6000": "/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4/hw/rtx_pro_6000.json" } }, "model": { "model_id": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", "min_vllm_version": "0.31.0", "docker_image": "vllm/vllm-openai:v0.31.0", "architecture": "moe", "parameter_count": "120B", "active_parameters": "12B", "context_length": 1048576, "base_args": [ "--trust-remote-code" ], "base_env": {}, "install": { "pip": { "command": "uv venv\nsource .venv/bin/activate\nuv pip install -U vllm --torch-backend auto" }, "docker": { "command": "docker pull vllm/vllm-openai:v0.31.0" } } }, "features": { "spec_decoding": { "description": "Use MTP, LiLiCorr on DGX Spark or DSpark", "default_mode": "mtp", "modes": { "mtp": { "label": "MTP", "description": "Draft three tokens on Hopper or five on Blackwell", "args": [ "--speculative-config.method", "mtp", "--speculative-config.num_speculative_tokens", "3" ], "hardware_overrides": { "blackwell": { "args": [ "--speculative-config.method", "mtp", "--speculative-config.num_speculative_tokens", "5" ] }, "dgx_spark_gb10": { "args": [ "--speculative-config.method", "mtp", "--speculative-config.num_speculative_tokens", "5", "--mamba-cache-mode", "align", "--chat-template-content-format", "string" ] } } }, "dspark": { "label": "DSpark", "description": "External draft with up to eight speculative tokens", "args": [ "--speculative-config", "{\"method\":\"dspark\",\"model\":\"RedHatAI/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B.dspark\",\"num_speculative_tokens\":8,\"draft_tensor_parallel_size\":1,\"draft_sample_method\":\"greedy\"}" ], "hardware_overrides": { "h100": { "args": [ "--speculative-config", "{\"method\":\"dspark\",\"model\":\"RedHatAI/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B.dspark\",\"num_speculative_tokens\":8,\"draft_tensor_parallel_size\":1,\"draft_sample_method\":\"greedy\"}", "--gpu-memory-utilization", "0.89", "--prefix-match-unit", "64", "--block-size", "64", "--compilation-config", "{\"max_cudagraph_capture_size\":192}", "--enable-expert-parallel", "--max-num-seqs", "128", "--max-model-len", "114688", "--max-num-batched-tokens", "16384", "--enable-prefix-caching", "--async-scheduling", "--mamba-backend", "flashinfer", "--mamba-ssm-cache-dtype", "float16", "--enable-mamba-cache-stochastic-rounding", "--mamba-cache-philox-rounds", "5", "--chat-template-content-format", "string", "--mamba-cache-mode", "align" ] } } }, "lillicorr": { "label": "LiLiCorr", "description": "Separate DFlash draft checkpoint, seven tokens on DGX Spark", "hardware": { "h100": "unsupported", "h200": "unsupported", "gb200": "unsupported", "rtx_pro_6000": "unsupported" }, "variants": [ "default" ], "args": [ "--speculative-config.model", "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr", "--speculative-config.num_speculative_tokens", "7", "--speculative-config.method", "dflash", "--max-num-seqs", "1", "--gpu-memory-utilization", "0.75", "--max-num-batched-tokens", "8192" ] } } }, "tool_calling": { "description": "Parse tool calls with qwen3_coder and automatic tool choice", "args": [ "--enable-auto-tool-choice", "--tool-call-parser", "qwen3_coder" ] }, "reasoning": { "description": "Separate Nemotron v3 reasoning from the final answer", "args": [ "--reasoning-parser", "nemotron_v3" ] }, "local_media": { "label": "Local Media", "description": "Allow access to host paths as in the BF16 cookbook examples; use only for trusted inputs", "args": [ "--allowed-local-media-path", "/" ] } }, "opt_in_features": [ "local_media" ], "variants": { "default": { "precision": "nvfp4", "precision_hardware": { "brand": "NVIDIA" }, "vram_minimum_gb": 91, "description": "NVFP4 W4A16 checkpoint; select a hardware profile for its cookbook configuration", "supported_hardware": [ "h100", "h200", "gb200", "dgx_spark_gb10", "rtx_pro_6000" ], "tp": { "h100": 2, "h200": 1, "gb200": 1, "dgx_spark_gb10": 1, "rtx_pro_6000": 1 }, "hardware_overrides": { "hopper": { "extra_args": [ "--max-model-len", "114688", "--max-num-seqs", "128", "--gpu-memory-utilization", "0.89", "--max-num-batched-tokens", "16384", "--block-size", "64", "--prefix-match-unit", "64", "--compilation_config.max_cudagraph_capture_size", "256", "--mamba-backend", "flashinfer", "--mamba-ssm-cache-dtype", "float16", "--enable-mamba-cache-stochastic-rounding", "--mamba-cache-philox-rounds", "5", "--enable-mamba-shared-prefix-checkpoint", "--mamba-cache-mode", "align", "--video-pruning-rate", "0.5", "--media-io-kwargs", "{\"video\":{\"fps\":2,\"num_frames\":256}}" ] }, "gb200": { "extra_args": [ "--max-num-seqs", "512", "--max-model-len", "114688", "--enable-prefix-caching", "--max-num-batched-tokens", "32768", "--prefix-match-unit", "16", "--enable-mamba-shared-prefix-checkpoint", "--async-scheduling", "--kernel-config", "{\"linear_backend_per_quant\":{\"fp8_w8a8\":\"torch\"}}", "--compilation-config", "{\"pass_config\":{\"fuse_attn_quant\":true},\"use_inductor_graph_partition\":true,\"cudagraph_mode\":\"FULL_AND_PIECEWISE\"}", "--mamba-backend", "flashinfer", "--mamba-ssm-cache-dtype", "float16", "--enable-mamba-cache-stochastic-rounding", "--mamba-cache-philox-rounds", "5", "--chat-template-content-format", "string", "--mamba-cache-mode", "align", "--video-pruning-rate", "0.5", "--media-io-kwargs", "{\"video\":{\"fps\":2,\"num_frames\":256}}" ] }, "dgx_spark_gb10": { "extra_args": [ "--max-num-seqs", "2", "--max-model-len", "114688", "--gpu-memory-utilization", "0.7", "--mamba-backend", "flashinfer", "--mamba-ssm-cache-dtype", "float16", "--enable-mamba-cache-stochastic-rounding", "--mamba-cache-philox-rounds", "5", "--prefix-match-unit", "16", "--enable-mamba-shared-prefix-checkpoint" ] }, "rtx_pro_6000": { "extra_args": [ "--max-model-len", "16384", "--max-num-seqs", "128" ] } } }, "fp8": { "model_id": "RedHatAI/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-FP8", "precision": "fp8", "vram_minimum_gb": 155, "description": "FP8 Dynamic W8A8; model-card baseline uses 2x H100 with MTP or DSpark", "tp": { "hopper": 2, "blackwell": 1 }, "extra_args": [ "--gpu-memory-utilization", "0.89", "--prefix-match-unit", "64", "--block-size", "64", "--compilation-config", "{\"max_cudagraph_capture_size\":192}", "--enable-expert-parallel", "--max-num-seqs", "128", "--max-model-len", "114688", "--max-num-batched-tokens", "16384", "--enable-prefix-caching", "--async-scheduling", "--mamba-backend", "flashinfer", "--mamba-ssm-cache-dtype", "float16", "--enable-mamba-cache-stochastic-rounding", "--mamba-cache-philox-rounds", "5", "--chat-template-content-format", "string", "--mamba-cache-mode", "align", "--enable-mamba-shared-prefix-checkpoint" ], "json": "/RedHatAI/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-FP8.json" }, "bf16": { "model_id": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16", "precision": "bf16", "vram_minimum_gb": 299, "description": "BF16 multimodal checkpoint on 4x H100 or 2x GB200", "supported_hardware": [ "h100", "gb200" ], "tp": { "h100": 4, "gb200": 2 }, "hardware_overrides": { "h100": { "extra_args": [ "--max-model-len", "114688", "--gpu-memory-utilization", "0.85", "--max-num-seqs", "32", "--limit-mm-per-prompt", "{\"video\":1,\"image\":1}", "--video-pruning-rate", "0.5", "--media-io-kwargs", "{\"video\":{\"fps\":2,\"num_frames\":256}}" ] }, "gb200": { "extra_args": [ "--max-model-len", "114688", "--async-scheduling", "--max-num-seqs", "32", "--limit-mm-per-prompt", "{\"video\":1,\"image\":1}", "--video-pruning-rate", "0.5", "--media-io-kwargs", "{\"video\":{\"fps\":2,\"num_frames\":256}}" ] } }, "json": "/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16.json" }, "nvfp4_gb200_4x": { "model_id": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4", "precision": "nvfp4", "precision_hardware": { "brand": "NVIDIA" }, "vram_minimum_gb": 91, "label": "NVFP4 · 4x GB200", "description": "Four-GPU text-only expert-parallel configuration", "supported_hardware": [ "gb200" ], "tp": 4, "extra_args": [ "--enable-expert-parallel", "--max-num-seqs", "512", "--max-model-len", "28672", "--no-enable-prefix-caching", "--async-scheduling", "--mamba-backend", "flashinfer", "--mamba-ssm-cache-dtype", "float16", "--enable-mamba-cache-stochastic-rounding", "--mamba-cache-philox-rounds", "5", "--chat-template-content-format", "string", "--mamba-cache-mode", "align" ] }, "bf16_gb200_4x": { "model_id": "nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16", "precision": "bf16", "vram_minimum_gb": 299, "label": "BF16 · 4x GB200", "description": "Four-GPU text-only expert-parallel configuration with float32 Mamba cache", "supported_hardware": [ "gb200" ], "tp": 4, "extra_args": [ "--enable-expert-parallel", "--max-num-seqs", "512", "--max-model-len", "28672", "--no-enable-prefix-caching", "--async-scheduling", "--mamba-backend", "flashinfer", "--mamba-ssm-cache-dtype", "float32", "--chat-template-content-format", "string", "--mamba-cache-mode", "align" ], "json": "/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16.json" } }, "hardware_overrides": {}, "guide": "## Overview\n\nNemotron 3.5 Super VL is a hybrid Mamba-MoE vision-language model with image and\nvideo input. The profiles below follow the NVIDIA cookbook's configurations on\n`vllm/vllm-openai:v0.31.0`. The page opens on the one-H200 NVFP4 configuration.\nEvery generated command uses the full Hugging Face model ID as its served name.\n\n## Prerequisites\n\n- Pull the pinned vLLM image and make the selected checkpoint available from\n Hugging Face. LiLiCorr on DGX Spark also needs its separate draft checkpoint.\n- The NVFP4 checkpoint's safetensors index reports 75.01 GB; the recipe's 91 GB\n minimum includes the schema's 20% allowance. The BF16 index reports 248.91 GB,\n giving a 299 GB minimum by the same method.\n- These are weight allowances, not a guarantee that every context and batch\n setting fits. Use the cookbook's per-hardware memory settings.\n\n## Hardware configurations\n\n| Hardware | Variant | TP | Context | Cookbook workload |\n| --- | --- | ---: | ---: | --- |\n| 1x H200 | NVFP4 | 1 | 114,688 | Multimodal, MTP 3 |\n| 2x H100 | NVFP4 | 2 | 114,688 | Multimodal, MTP 3 |\n| 4x H100 | BF16 | 4 | 114,688 | Multimodal, MTP 3 |\n| 1x GB200 | NVFP4 | 1 | 114,688 | Text only, MTP 5 |\n| 2x GB200 | BF16 | 2 | 114,688 | Multimodal, MTP 5 |\n| 4x GB200 | NVFP4 or BF16 4x | 4 | 28,672 | Text only, MTP 5 |\n| 1x DGX Spark | NVFP4 | 1 | 114,688 | Text-only MTP 5 or LiLiCorr 7 |\n| 1x RTX Pro 6000 | NVFP4 | 1 | 16,384 | MTP 5 |\n\nSelect the matching Hardware and Variant pills to generate these commands.\nOn GB200, choose BF16 with TP 2 for image or video input: the 1x NVFP4 and\n4x variants use `--chat-template-content-format string` for text workloads.\nDGX Spark defaults to its text-only MTP command; the LiLiCorr mode uses the\nseparate DFlash draft checkpoint and changes its concurrency and memory budget.\nThe BF16 notebook examples allow local media paths under `/`; turn on the\noptional Local Media feature only when serving trusted clients that need it.\n\nThe cookbook also validates NVFP4 on 2x A100 with an 8,192-token allocation.\nA100 is not in the site's hardware taxonomy, so use its command directly:\n\n```bash\nvllm serve nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4 \\\n --served-model-name nemotron-3.5-super \\\n --trust-remote-code \\\n --tensor-parallel-size 2 \\\n --pipeline-parallel-size 1 \\\n --data-parallel-size 1 \\\n --max-model-len 8192 \\\n --max-num-seqs 512 \\\n --speculative_config.method mtp \\\n --speculative_config.num_speculative_tokens 5 \\\n --enable-auto-tool-choice \\\n --tool-call-parser qwen3_coder \\\n --reasoning-parser nemotron_v3\n```\n\nUse `nemotron-3.5-super` as the client model name for this A100 command.\n\nThe cookbook also provides 1M-token text-only commands. They use\n`--language-model-only` and `VLLM_ALLOW_LONG_MAX_MODEL_LEN=1`; apply those\ncomplete commands from the notebook when the full window is needed. The\nselectable configurations here retain shorter hardware-specific allocations.\n\n## Client usage\n\nSend requests to `http://127.0.0.1:8000/v1` with model\n`nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4` or the BF16 model ID,\ndepending on the selected variant. The cookbook includes image and video\nrequests, tool calls, and reasoning output examples.\n\n## References\n\n- [NVIDIA vLLM cookbook](https://github.com/NVIDIA-NeMo/Nemotron/blob/main/usage-cookbook/Nemotron-3.5-Super-VL/vllm_cookbook.ipynb)\n- [NVFP4 checkpoint](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4)\n- [BF16 checkpoint](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-BF16)\n- [LiLiCorr draft checkpoint](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4-LiLiCorr)\n- [RedHatAI FP8 model card](https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-FP8)\n- [RedHatAI DSpark model card](https://huggingface.co/RedHatAI/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B.dspark)\n", "source": { "source": "vllm", "repo": "vllm-project/vllm", "ref": "main", "sha": "389ffebe555705c85337025d5df108d0384d365e", "path": "recipes/models/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4.yaml", "edit_url": "https://github.com/vllm-project/vllm/edit/main/recipes/models/nvidia/NVIDIA-Nemotron-3.5-Super-VL-120B-A12B-NVFP4.yaml", "overlays": [] } }